From 5619f1b3954aba2c956153f64d3f375bc097ae47 Mon Sep 17 00:00:00 2001
From: Michal Sek <michal.sek@swmansion.com>
Date: Fri, 29 Nov 2024 16:54:18 +0100
Subject: [PATCH 1/2] feat: working on open ai example

---
 .gitignore                                    |   3 +
 .../common-app/src/examples/OpenAI/OpenAI.tsx | 114 ++++++++++++++++++
 apps/common-app/src/examples/OpenAI/index.ts  |   1 +
 apps/common-app/src/examples/index.ts         |   8 ++
 apps/common-app/src/utils/env.ts              |   3 +
 apps/fabric-example/babel.config.js           |   2 +-
 apps/fabric-example/ios/Podfile.lock          |   4 +-
 apps/fabric-example/package.json              |   1 +
 yarn.lock                                     |  19 +++
 9 files changed, 152 insertions(+), 3 deletions(-)
 create mode 100644 apps/common-app/src/examples/OpenAI/OpenAI.tsx
 create mode 100644 apps/common-app/src/examples/OpenAI/index.ts
 create mode 100644 apps/common-app/src/utils/env.ts

diff --git a/.gitignore b/.gitignore
index 6d08e317..4bf942bb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -83,3 +83,6 @@ react-native-audio-api*.tgz
 # Android
 .kotlin
 
+
+# Envs
+.env
diff --git a/apps/common-app/src/examples/OpenAI/OpenAI.tsx b/apps/common-app/src/examples/OpenAI/OpenAI.tsx
new file mode 100644
index 00000000..311f143c
--- /dev/null
+++ b/apps/common-app/src/examples/OpenAI/OpenAI.tsx
@@ -0,0 +1,114 @@
+import React, { useState, FC } from 'react';
+import { AudioBuffer, AudioContext } from 'react-native-audio-api';
+import { ActivityIndicator, TextInput, StyleSheet } from 'react-native';
+
+import { Container, Button, Spacer } from '../../components';
+import Env from '../../utils/env';
+import { colors } from '../../styles';
+
+async function getOpenAIResponse(input: string, voice: string = 'alloy') {
+  return await fetch('https://api.openai.com/v1/audio/speech', {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${Env.openAiToken}`,
+    },
+    body: JSON.stringify({
+      model: 'tts-1-hd',
+      voice: voice,
+      input: input,
+      response_format: 'pcm',
+    }),
+  }).then((response) => response.arrayBuffer());
+}
+
+function goofyResample(
+  audioContext: AudioContext,
+  input: Int16Array
+): AudioBuffer {
+  const outputBuffer = audioContext.createBuffer(2, input.length * 2, 48000);
+  const processingChannel: Array<number> = [];
+  const upSampleChannel: Array<number> = [];
+
+  for (let i = 0; i < input.length; i += 1) {
+    processingChannel[i] = input[i] / 32768.0;
+  }
+
+  for (let i = 0; i < input.length; i += 1) {
+    const isLast = i === input.length - 1;
+    const currentSample = processingChannel[i];
+    const nextSample = isLast ? currentSample : processingChannel[i + 1];
+
+    upSampleChannel[2 * i] = currentSample;
+    upSampleChannel[2 * i + 1] = (currentSample + nextSample) / 2;
+  }
+
+  outputBuffer.copyToChannel(upSampleChannel, 0);
+  outputBuffer.copyToChannel(upSampleChannel, 1);
+
+  return outputBuffer;
+}
+
+const OpenAI: FC = () => {
+  const [isLoading, setIsLoading] = useState(false);
+  const [textToRead, setTextToRead] = useState('');
+
+  const onTestOpenAI = async () => {
+    if (isLoading) {
+      return;
+    }
+
+    const aCtx = new AudioContext();
+
+    setIsLoading(true);
+    const results = await getOpenAIResponse(textToRead, 'alloy');
+    setIsLoading(false);
+
+    const audioBuffer = goofyResample(aCtx, new Int16Array(results));
+    const sourceNode = aCtx.createBufferSource();
+    const duration = audioBuffer.duration;
+    const now = aCtx.currentTime;
+
+    sourceNode.buffer = audioBuffer;
+
+    sourceNode.connect(aCtx.destination);
+
+    sourceNode.start(now);
+    sourceNode.stop(now + duration);
+  };
+
+  return (
+    <Container style={styles.container}>
+      <Spacer.Vertical size={60} />
+      <TextInput
+        value={textToRead}
+        onChangeText={setTextToRead}
+        style={styles.textInput}
+        multiline
+      />
+      <Spacer.Vertical size={24} />
+      <Button onPress={onTestOpenAI} title="Test OpenAI" />
+      <Spacer.Vertical size={24} />
+      {isLoading && <ActivityIndicator />}
+    </Container>
+  );
+};
+
+export default OpenAI;
+
+const styles = StyleSheet.create({
+  container: {
+    alignItems: 'center',
+  },
+  textInput: {
+    backgroundColor: 'transparent',
+    borderColor: colors.border,
+    color: colors.white,
+    borderWidth: 1,
+    fontSize: 16,
+    padding: 16,
+    width: 280,
+    height: 200,
+    borderRadius: 6,
+  },
+});
diff --git a/apps/common-app/src/examples/OpenAI/index.ts b/apps/common-app/src/examples/OpenAI/index.ts
new file mode 100644
index 00000000..05dda66c
--- /dev/null
+++ b/apps/common-app/src/examples/OpenAI/index.ts
@@ -0,0 +1 @@
+export { default } from './OpenAI';
diff --git a/apps/common-app/src/examples/index.ts b/apps/common-app/src/examples/index.ts
index 2ce044a6..78bac747 100644
--- a/apps/common-app/src/examples/index.ts
+++ b/apps/common-app/src/examples/index.ts
@@ -1,6 +1,7 @@
 import { StackNavigationProp } from '@react-navigation/stack';
 
 import Piano from './Piano';
+import OpenAI from './OpenAI';
 import Metronome from './Metronome';
 import Oscillator from './Oscillator';
 import DrumMachine from './DrumMachine';
@@ -10,6 +11,7 @@ type NavigationParamList = {
   Metronome: undefined;
   DrumMachine: undefined;
   Piano: undefined;
+  OpenAI: undefined;
 };
 
 export type ExampleKey = keyof NavigationParamList;
@@ -47,4 +49,10 @@ export const Examples: Example[] = [
     subtitle: 'Generate sound waves',
     screen: Oscillator,
   },
+  {
+    key: 'OpenAI',
+    title: 'OpenAI',
+    subtitle: 'Test OpenAI',
+    screen: OpenAI,
+  },
 ] as const;
diff --git a/apps/common-app/src/utils/env.ts b/apps/common-app/src/utils/env.ts
new file mode 100644
index 00000000..b39f8245
--- /dev/null
+++ b/apps/common-app/src/utils/env.ts
@@ -0,0 +1,3 @@
+export default {
+  openAiToken: process.env.OPENAI_API_TOKEN,
+};
diff --git a/apps/fabric-example/babel.config.js b/apps/fabric-example/babel.config.js
index 02c7d135..d9adb13e 100644
--- a/apps/fabric-example/babel.config.js
+++ b/apps/fabric-example/babel.config.js
@@ -1,4 +1,4 @@
 module.exports = {
   presets: ['module:@react-native/babel-preset'],
-  plugins: ['react-native-reanimated/plugin'],
+  plugins: ['react-native-reanimated/plugin', 'module:react-native-dotenv'],
 };
diff --git a/apps/fabric-example/ios/Podfile.lock b/apps/fabric-example/ios/Podfile.lock
index 96960d34..4944dd17 100644
--- a/apps/fabric-example/ios/Podfile.lock
+++ b/apps/fabric-example/ios/Podfile.lock
@@ -1598,7 +1598,7 @@ PODS:
     - React-logger (= 0.76.0)
     - React-perflogger (= 0.76.0)
     - React-utils (= 0.76.0)
-  - RNAudioAPI (0.1.0):
+  - RNAudioAPI (0.2.0):
     - DoubleConversion
     - glog
     - hermes-engine
@@ -2058,7 +2058,7 @@ SPEC CHECKSUMS:
   React-utils: d9624101245ebaab39c9f1bd786132da0b4f27ff
   ReactCodegen: dbfef1fef26f42c900bb1884fa149d49d501d64d
   ReactCommon: 429ca28cd813c31359c73ffac6dc24f93347d522
-  RNAudioAPI: a4068f739b4e80e636ac6110d70858ea89ac4835
+  RNAudioAPI: adb400fcab1b85cafaff4e0880be5d4616fd4727
   RNGestureHandler: a3822e519fd1e9885b92d8c9bd82a7a0be305fe3
   RNReanimated: 77242c6d67416988a2fd9f5cf574bb3e60016362
   RNScreens: e389d6a6a66a4f0d3662924ecae803073ccce8ec
diff --git a/apps/fabric-example/package.json b/apps/fabric-example/package.json
index 5e54d8cf..ffb1c2cf 100644
--- a/apps/fabric-example/package.json
+++ b/apps/fabric-example/package.json
@@ -42,6 +42,7 @@
     "eslint": "^8.19.0",
     "jest": "^29.6.3",
     "prettier": "2.8.8",
+    "react-native-dotenv": "^3.4.11",
     "react-test-renderer": "18.3.1",
     "typescript": "5.0.4"
   },
diff --git a/yarn.lock b/yarn.lock
index 3dc3aa11..6cdb51c6 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5418,6 +5418,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"dotenv@npm:^16.4.5":
+  version: 16.4.5
+  resolution: "dotenv@npm:16.4.5"
+  checksum: 10/55a3134601115194ae0f924e54473459ed0d9fc340ae610b676e248cca45aa7c680d86365318ea964e6da4e2ea80c4514c1adab5adb43d6867fb57ff068f95c8
+  languageName: node
+  linkType: hard
+
 "eastasianwidth@npm:^0.2.0":
   version: 0.2.0
   resolution: "eastasianwidth@npm:0.2.0"
@@ -6150,6 +6157,7 @@ __metadata:
     react-dom: "npm:18.2.0"
     react-native: "npm:0.76.0"
     react-native-audio-api: "workspace:*"
+    react-native-dotenv: "npm:^3.4.11"
     react-native-gesture-handler: "npm:^2.20.2"
     react-native-reanimated: "npm:^3.16.1"
     react-native-safe-area-context: "npm:^4.12.0"
@@ -10219,6 +10227,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"react-native-dotenv@npm:^3.4.11":
+  version: 3.4.11
+  resolution: "react-native-dotenv@npm:3.4.11"
+  dependencies:
+    dotenv: "npm:^16.4.5"
+  peerDependencies:
+    "@babel/runtime": ^7.20.6
+  checksum: 10/09e8a7310fcb01ac021e71db9328e9d342d1e117bf68026b12de0392bfe17292ac6a071f03b88e7fb42c82a8f2fdf03bc520c7dedd2f80a1448cb3de5e03d4fb
+  languageName: node
+  linkType: hard
+
 "react-native-gesture-handler@npm:^2.20.2":
   version: 2.21.1
   resolution: "react-native-gesture-handler@npm:2.21.1"

From cb1646d1b69ae4fbd8136baaa27607d788ae1394 Mon Sep 17 00:00:00 2001
From: Michal Sek <michal.sek@swmansion.com>
Date: Thu, 5 Dec 2024 12:56:57 +0100
Subject: [PATCH 2/2] feat: cleanup and renaming

---
 apps/common-app/src/examples/OpenAI/index.ts  |  1 -
 .../TextToSpeech.tsx}                         | 23 ++++++++++++++-----
 .../src/examples/TextToSpeech/index.ts        |  1 +
 apps/common-app/src/examples/index.ts         | 16 ++++++-------
 4 files changed, 26 insertions(+), 15 deletions(-)
 delete mode 100644 apps/common-app/src/examples/OpenAI/index.ts
 rename apps/common-app/src/examples/{OpenAI/OpenAI.tsx => TextToSpeech/TextToSpeech.tsx} (84%)
 create mode 100644 apps/common-app/src/examples/TextToSpeech/index.ts

diff --git a/apps/common-app/src/examples/OpenAI/index.ts b/apps/common-app/src/examples/OpenAI/index.ts
deleted file mode 100644
index 05dda66c..00000000
--- a/apps/common-app/src/examples/OpenAI/index.ts
+++ /dev/null
@@ -1 +0,0 @@
-export { default } from './OpenAI';
diff --git a/apps/common-app/src/examples/OpenAI/OpenAI.tsx b/apps/common-app/src/examples/TextToSpeech/TextToSpeech.tsx
similarity index 84%
rename from apps/common-app/src/examples/OpenAI/OpenAI.tsx
rename to apps/common-app/src/examples/TextToSpeech/TextToSpeech.tsx
index 311f143c..7dd96d1a 100644
--- a/apps/common-app/src/examples/OpenAI/OpenAI.tsx
+++ b/apps/common-app/src/examples/TextToSpeech/TextToSpeech.tsx
@@ -22,16 +22,27 @@ async function getOpenAIResponse(input: string, voice: string = 'alloy') {
   }).then((response) => response.arrayBuffer());
 }
 
+const openAISampleRate = 24000;
+const maxInputValue = 32768.0;
+
+// TODO: this should ideally be done using native code through .decodeAudioData
 function goofyResample(
   audioContext: AudioContext,
   input: Int16Array
 ): AudioBuffer {
-  const outputBuffer = audioContext.createBuffer(2, input.length * 2, 48000);
+  const scale = audioContext.sampleRate / openAISampleRate;
+
+  const outputBuffer = audioContext.createBuffer(
+    2,
+    input.length * scale,
+    audioContext.sampleRate
+  );
+
   const processingChannel: Array<number> = [];
   const upSampleChannel: Array<number> = [];
 
   for (let i = 0; i < input.length; i += 1) {
-    processingChannel[i] = input[i] / 32768.0;
+    processingChannel[i] = input[i] / maxInputValue;
   }
 
   for (let i = 0; i < input.length; i += 1) {
@@ -49,11 +60,11 @@ function goofyResample(
   return outputBuffer;
 }
 
-const OpenAI: FC = () => {
+const TextToSpeech: FC = () => {
   const [isLoading, setIsLoading] = useState(false);
   const [textToRead, setTextToRead] = useState('');
 
-  const onTestOpenAI = async () => {
+  const onReadText = async () => {
     if (isLoading) {
       return;
     }
@@ -87,14 +98,14 @@ const OpenAI: FC = () => {
         multiline
       />
       <Spacer.Vertical size={24} />
-      <Button onPress={onTestOpenAI} title="Test OpenAI" />
+      <Button onPress={onReadText} title="Read Text" />
       <Spacer.Vertical size={24} />
       {isLoading && <ActivityIndicator />}
     </Container>
   );
 };
 
-export default OpenAI;
+export default TextToSpeech;
 
 const styles = StyleSheet.create({
   container: {
diff --git a/apps/common-app/src/examples/TextToSpeech/index.ts b/apps/common-app/src/examples/TextToSpeech/index.ts
new file mode 100644
index 00000000..506026bc
--- /dev/null
+++ b/apps/common-app/src/examples/TextToSpeech/index.ts
@@ -0,0 +1 @@
+export { default } from './TextToSpeech';
diff --git a/apps/common-app/src/examples/index.ts b/apps/common-app/src/examples/index.ts
index a4318b24..0fc8bf29 100644
--- a/apps/common-app/src/examples/index.ts
+++ b/apps/common-app/src/examples/index.ts
@@ -1,7 +1,7 @@
 import { StackNavigationProp } from '@react-navigation/stack';
 
 import Piano from './Piano';
-import OpenAI from './OpenAI';
+import TextToSpeech from './TextToSpeech';
 import Metronome from './Metronome';
 import Oscillator from './Oscillator';
 import DrumMachine from './DrumMachine';
@@ -12,7 +12,7 @@ type NavigationParamList = {
   Metronome: undefined;
   DrumMachine: undefined;
   Piano: undefined;
-  OpenAI: undefined;
+  TextToSpeech: undefined;
   AudioFile: undefined;
 };
 
@@ -39,6 +39,12 @@ export const Examples: Example[] = [
     subtitle: 'Play some notes',
     screen: Piano,
   },
+  {
+    key: 'TextToSpeech',
+    title: 'Text to Speech',
+    subtitle: 'type some text and hear it spoken',
+    screen: TextToSpeech,
+  },
   {
     key: 'Metronome',
     title: 'Metronome',
@@ -51,12 +57,6 @@ export const Examples: Example[] = [
     subtitle: 'Generate sound waves',
     screen: Oscillator,
   },
-  {
-    key: 'OpenAI',
-    title: 'OpenAI',
-    subtitle: 'Test OpenAI',
-    screen: OpenAI,
-  },
   {
     key: 'AudioFile',
     title: 'Audio File',