From 2d62a4d869ff19ff41ed191903ccfdaf44741378 Mon Sep 17 00:00:00 2001 From: Gilad Dekel Date: Thu, 21 Sep 2023 16:46:23 +0300 Subject: [PATCH 1/6] fixture/add-client-side-http-connectors-to-other-llms --- src/components/chat-window.vue | 19 +- src/scripts/ts/modules/altGPT.ts | 3 +- src/scripts/ts/modules/openAI.ts | 371 ++++++++++++++++++++++++++----- 3 files changed, 330 insertions(+), 63 deletions(-) diff --git a/src/components/chat-window.vue b/src/components/chat-window.vue index dda64af..78d0279 100644 --- a/src/components/chat-window.vue +++ b/src/components/chat-window.vue @@ -163,9 +163,9 @@ export default { if (this.isActive) return; const msg = this.userMessage; console.log('submit: ', msg); - + this.isLoading = true; this.$forceUpdate(); - + // if (this.selectedPlugins?.length > 0) { // try{ // console.log('chat:submit: Selected plugins: ', this.selectedPlugins); @@ -191,7 +191,7 @@ export default { // } try{ - const config = { + const config = { frequency_penalty: this.bot?.frequency_penalty, presence_penalty: this.bot?.presence_penalty, temperature: this.bot?.temperature, @@ -214,7 +214,7 @@ export default { this.userMessage = ''; this.messages.push(newUserMsg) } - + const newMsg = {role:'assistant', content: ''}; this.messages.push(newMsg); this.scrollChatToBottom(); @@ -230,11 +230,17 @@ export default { if (this.selectedPlugins?.length > 0) { await this.altGpt.performSmartCompletion(messages, this.selectedPlugins, config, onDelta, priming, this.pluginsSettings); } else { - await this.openAI.createChatCompletionStream(messages, config, priming, onDelta); + // await this.openAI.createChatCompletionStream(messages, config, priming, onDelta); + const httpResponse = await this.openAI.createChatCompletionRes( + messages, + config, + priming, + onDelta + ); } this.$forceUpdate(); - + this.messages.push(this.messages.pop()); // so it'll be stored properly in cache // newText = newMsg.content; } catch(err) { @@ -334,5 +340,4 @@ export default { p { margin-top:0; margin-bottom:0; } img { min-width:40px; min-height:40px; border: 1px solid #999; background-color:#555 ; } } - diff --git a/src/scripts/ts/modules/altGPT.ts b/src/scripts/ts/modules/altGPT.ts index d347c1b..cc0318b 100644 --- a/src/scripts/ts/modules/altGPT.ts +++ b/src/scripts/ts/modules/altGPT.ts @@ -172,8 +172,7 @@ export class AltGPT { }); const _priming = `You are a bot designed to assist machines in answering user inquiries based solely on the given context. If relevant to user's inquiry, extract full links from the context and maintain them unchanged. If there are multiple outcomes, present them as bullet points, each accompanied by the pertinent link. If the supplied context is empty or yields no results, state that the search produced no findings and recommend refining the query. If the answer is not included, respond with 'Hmm, I'm not sure...'. ${priming}`; const _config = { ...this.basicConfig, ...config }; - const res = await this.openAI.createChatCompletionStream(_messages, _config, _priming, onDelta); - + const res = await this.openAI.createChatCompletionRes(_messages, _config, _priming, onDelta); return res; } diff --git a/src/scripts/ts/modules/openAI.ts b/src/scripts/ts/modules/openAI.ts index c1afc30..2b7f04b 100644 --- a/src/scripts/ts/modules/openAI.ts +++ b/src/scripts/ts/modules/openAI.ts @@ -83,81 +83,344 @@ export class OpenAI { return json ?? res; } - public async createChatCompletionStream(messages: IConvMessage[], config, priming?: string, onDelta?: (data) => void, onProgress?: (wholeData) => void) { - const p = libx.newPromise(); - const url = `https://api.openai.com/v1/chat/completions`; - const apiKey = config.apikey; - const payload = { - "messages": [ - { - role: 'system', - content: priming ?? `You are an assistant bot.`, - }, - ...messages - ], - ...{ - // defaults: - frequency_penalty: 0, - presence_penalty: 0, - temperature: 0.8, - max_tokens: 256, // 512, - model: "gpt-3.5-turbo", - ...config, // override - stream: true, - // user: 'userId', - top_p: 1.0, - n: 1, - } - }; - delete payload.apikey; + // public async createChatCompletionStream(messages: IConvMessage[], config, priming?: string, onDelta?: (data) => void, onProgress?: (wholeData) => void) { + // const p = libx.newPromise(); + // const url = `https://api.openai.com/v1/chat/completions`; + // const apiKey = config.apikey; + // const payload = { + // "messages": [ + // { + // role: 'system', + // content: priming ?? `You are an assistant bot.`, + // }, + // ...messages + // ], + // ...{ + // // defaults: + // frequency_penalty: 0, + // presence_penalty: 0, + // temperature: 0.8, + // max_tokens: 256, // 512, + // model: "gpt-3.5-turbo", + // ...config, // override + // stream: true, + // // user: 'userId', + // top_p: 1.0, + // n: 1, + // } + // }; + // delete payload.apikey; + // const eventSource = new SSE(url, { + // method: "POST", + // headers: { + // 'Accept': 'application/json, text/plain, */*', + // 'Authorization': `Bearer ${apiKey}`, + // 'Content-Type': 'application/json', + // }, + // payload: JSON.stringify({ + // ...payload, + // "stream": true, + // }), + // }) as SSE; + + // let contents = ''; + + // eventSource.addEventListener('error', (event: any) => { + // if (!contents) { + // libx.log.e('error: ', event, contents) + // p.reject(JSON.parse(event.data)); + // } + // }); + + // eventSource.addEventListener('message', async (event: any) => { + // if (event.data === '[DONE]') { + // libx.log.d('message: done: ', event, contents) + // p.resolve(contents); + // return; + // } + + // try { + // const chunk = this.parseChunk(event.data); + // if (chunk.choices && chunk.choices.length > 0) { + // const newData = chunk.choices[0]?.delta?.content || ''; + // contents += newData; + // if (onDelta) onDelta(newData); + // if (onProgress) onProgress(contents); + // } + // } catch (err) { + // console.error(err); + // p.reject(err); + // } + // }); + + // eventSource.stream(); + // return p; + // } + + + public async createChatCompletionStream({ + url, + headers, + payload, + p, + getNewData, + onDelta, + onProgress, + }) { const eventSource = new SSE(url, { method: "POST", - headers: { - 'Accept': 'application/json, text/plain, */*', - 'Authorization': `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - payload: JSON.stringify({ - ...payload, - "stream": true, - }), + headers: headers, + payload: JSON.stringify(payload), }) as SSE; - let contents = ''; + let contents = ""; - eventSource.addEventListener('error', (event: any) => { + eventSource.addEventListener("error", (event: any) => { if (!contents) { - libx.log.e('error: ', event, contents) - p.reject(JSON.parse(event.data)); + p.reject(event.data && JSON.parse(event.data)); } }); - eventSource.addEventListener('message', async (event: any) => { - if (event.data === '[DONE]') { - libx.log.d('message: done: ', event, contents) - p.resolve(contents); - return; - } - + eventSource.addEventListener("message", async (event: any) => { try { - const chunk = this.parseChunk(event.data); - if (chunk.choices && chunk.choices.length > 0) { - const newData = chunk.choices[0]?.delta?.content || ''; - contents += newData; - if (onDelta) onDelta(newData); - if (onProgress) onProgress(contents); + console.log("🚀 ~ event data:", event?.data); + + if (event?.data === "[DONE]") { + // Handle this special case, maybe close the event source or do some other logic + eventSource.close(); + return; + } + + if (!isValidJSON(event?.data)) { + console.error("Invalid JSON received:", event?.data); + return; } + + const chunk = JSON.parse(event?.data); + const newData = getNewData(chunk); + contents += newData; + + if (onDelta) onDelta(newData); + if (onProgress) onProgress(contents); } catch (err) { - console.error(err); p.reject(err); } }); + function isValidJSON(jsonString) { + try { + JSON.parse(jsonString); + return true; + } catch (err) { + return false; + } + } + eventSource.stream(); + return p; } + public async createChatCompletionHTTP({ + url, + headers, + payload, + getNewData, + onDelta, + }) { + const response = await fetch(url, { + method: "POST", + headers: headers, + body: JSON.stringify(payload), + }); + if (!response.ok) { + throw new Error(`HTTP error! Status: ${response.status}`); + } + let httpResponse = await response.json(); + let res = getNewData(httpResponse); + return onDelta(res); // Assuming httpResponse has a 'content' field + } + + public async createChatCompletionRes( + messages: IConvMessage[], + config, + priming?: string, + onDelta?: (data) => void, + onProgress?: (wholeData) => void + ) { + const p = libx.newPromise(); + + //// payload + + const setupPayloadForGPT = () => { + return { + messages: [ + { + role: "system", + content: priming ?? `You are an assistant bot.`, + }, + ...messages, + ], + ...{ + frequency_penalty: 0, + presence_penalty: 0, + temperature: 0.8, + max_tokens: 256, + model: "gpt-3.5-turbo", + ...config, + stream: true, + top_p: 1.0, + n: 1, + }, + }; + }; + + const setupPayloadForCohere = () => { + const promptContent = [ + priming ?? `You are an assistant bot.`, + ...messages.map((message) => message.content), + ].join("\n"); + return { + prompt: promptContent, + model: "command", + max_tokens: 256, + stream: true, + ...config, + }; + }; + + const setupPayloadForAnthropic = () => { + const promptContent = `\n\nHuman: ${messages + .map((message) => message.content) + .join("\n\nHuman: ")}\n\nAssistant:`; + return { + prompt: promptContent, + model: "claude-2", // or "claude-instant-1" + max_tokens_to_sample: 300, // or another value you prefer + ...config, // to handle other configuration parameters like temperature, top_p, etc. + }; + }; + + const setupPayloadForAI21 = () => { + return { + prompt: messages.map((message) => message.content).join("\n"), + model_type: config.model_type || "mid", // or 'light' or 'ultra' based on your choice + + maxTokens: 256, + ...config, + }; + }; + + /////////// handle responses + const handleGPTResponse = (chunk) => { + if (chunk.choices && chunk.choices.length > 0) { + console.log( + "🚀 ~ file: openAI.ts:404 ~ OpenAI ~ handleGPTResponse ~ chunk.choices[0]?.delta?.content:", + chunk.choices[0]?.delta?.content + ); + return chunk.choices[0]?.delta?.content || ""; + } + return ""; + }; + + const handleCohereResponse = (chunk) => { + // For simplicity, I'm assuming you only request one generation. + // If you request multiple generations (using num_generations parameter > 1), + // you may want to handle all the returned texts appropriately. + if (chunk.generations && chunk.generations.length > 0) { + return chunk.generations[0].text; + } + throw new Error("No completion found in Cohere response."); + }; + const handleAnthropicResponse = (chunk) => { + return chunk.completion; + }; + + const handleAI21Response = (httpResponse) => { + return httpResponse?.completions[0]?.data?.text; + }; + + const getModelType = (model) => { + if (model.includes("gpt") || model.includes("davinci")) { + return "gpt"; + } else if (model.includes("xlarge")) { + return "cohere"; + } else if (model.includes("claude")) { + return "anthropic"; + } else if (model.includes("j2")) { + return "ai21"; + } + }; + + let modelType = getModelType(config.model); + let url: string; + let payload: any; + let getNewData: (data: string) => string; + let headers = { + Accept: "application/json, text/plain, */*", + "Content-Type": "application/json", + }; + let useSSE = false; + switch (modelType) { + case "gpt": + url = `https://api.openai.com/v1/chat/completions`; + payload = setupPayloadForGPT(); + getNewData = (data) => handleGPTResponse(data); + headers["Authorization"] = `Bearer ${config.apikey}`; + useSSE = true; + break; + case "cohere": + url = `https://api.cohere.ai/v1/generate`; + payload = setupPayloadForCohere(); + getNewData = (data) => handleCohereResponse(data); + headers["Authorization"] = `Bearer ${config.apikey}`; + break; + case "anthropic": + url = `https://api.anthropic.com/v1/complete`; + payload = setupPayloadForAnthropic(); + getNewData = (data) => handleAnthropicResponse(data); + headers["x-api-key"] = config.apikey; + headers["anthropic-version"] = "2023-06-01"; + break; + case "ai21": + url = `https://api.ai21.com/studio/v1/j2-${config.model_type || "mid" + }/complete`; + payload = setupPayloadForAI21(); + getNewData = (data) => handleAI21Response(data); + headers["Authorization"] = `Bearer ${config.apikey}`; + break; + // Add more cases for other models in the future as needed + default: + throw new Error("Unsupported model type."); + } + + delete payload.apikey; + + console.log("Config:", config); + console.log("Messages:", messages); + + if (useSSE) { + return this.createChatCompletionStream({ + url, + headers, + payload, + p, + getNewData, + onDelta, + onProgress, + }); + } else { + return this.createChatCompletionHTTP({ + url, + headers, + payload, + getNewData, + onDelta, + }); + } + } + private parseChunk(buffer: any): OpenAIResponseChunk { const chunk = buffer.toString().split(/\s*data\:\s*/); // for (let content of chunk) { From d80341c32f7018319d2829c4501b2e39f2321f50 Mon Sep 17 00:00:00 2001 From: Gilad Dekel Date: Thu, 21 Sep 2023 17:38:20 +0300 Subject: [PATCH 2/6] improve founction --- src/scripts/ts/modules/openAI.ts | 53 ++++++++++++-------------------- 1 file changed, 20 insertions(+), 33 deletions(-) diff --git a/src/scripts/ts/modules/openAI.ts b/src/scripts/ts/modules/openAI.ts index 2b7f04b..f983ea2 100644 --- a/src/scripts/ts/modules/openAI.ts +++ b/src/scripts/ts/modules/openAI.ts @@ -171,52 +171,43 @@ export class OpenAI { const eventSource = new SSE(url, { method: "POST", headers: headers, - payload: JSON.stringify(payload), + payload: JSON.stringify({ + ...payload, + stream: true, + }), }) as SSE; let contents = ""; eventSource.addEventListener("error", (event: any) => { if (!contents) { + libx.log.e("error: ", event, contents); p.reject(event.data && JSON.parse(event.data)); } }); + eventSource.addEventListener("message", async (event: any) => { + if (event?.data === "[DONE]") { + // Handle this special case, maybe close the event source or do some other logic + libx.log.d("message: done: ", event, contents); + p.resolve(contents); + // eventSource.close(); + return; + } try { - console.log("🚀 ~ event data:", event?.data); - - if (event?.data === "[DONE]") { - // Handle this special case, maybe close the event source or do some other logic - eventSource.close(); - return; - } - - if (!isValidJSON(event?.data)) { - console.error("Invalid JSON received:", event?.data); - return; + const chunk = this.parseChunk(event.data); + if (chunk.choices && chunk.choices.length > 0) { + const newData = getNewData(chunk); + contents += newData; + if (onDelta) onDelta(newData); + if (onProgress) onProgress(contents); } - - const chunk = JSON.parse(event?.data); - const newData = getNewData(chunk); - contents += newData; - - if (onDelta) onDelta(newData); - if (onProgress) onProgress(contents); } catch (err) { + console.error(err); p.reject(err); } }); - - function isValidJSON(jsonString) { - try { - JSON.parse(jsonString); - return true; - } catch (err) { - return false; - } - } - eventSource.stream(); return p; @@ -315,10 +306,6 @@ export class OpenAI { /////////// handle responses const handleGPTResponse = (chunk) => { if (chunk.choices && chunk.choices.length > 0) { - console.log( - "🚀 ~ file: openAI.ts:404 ~ OpenAI ~ handleGPTResponse ~ chunk.choices[0]?.delta?.content:", - chunk.choices[0]?.delta?.content - ); return chunk.choices[0]?.delta?.content || ""; } return ""; From 673f458b0cff789e6320f552a3e4e82efae01aea Mon Sep 17 00:00:00 2001 From: Gilad Dekel Date: Thu, 21 Sep 2023 23:35:25 +0300 Subject: [PATCH 3/6] improve setupPayloadForAI21 --- src/scripts/ts/modules/openAI.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/scripts/ts/modules/openAI.ts b/src/scripts/ts/modules/openAI.ts index f983ea2..b104594 100644 --- a/src/scripts/ts/modules/openAI.ts +++ b/src/scripts/ts/modules/openAI.ts @@ -295,9 +295,9 @@ export class OpenAI { const setupPayloadForAI21 = () => { return { - prompt: messages.map((message) => message.content).join("\n"), + // prompt: messages.map((message) => message.content).join("\n"), + prompt: messages.length > 0 ? messages[messages.length - 1].content : "", model_type: config.model_type || "mid", // or 'light' or 'ultra' based on your choice - maxTokens: 256, ...config, }; From 0360c0f90ee1e2517dca2dfcd746488788188104 Mon Sep 17 00:00:00 2001 From: Gilad Dekel Date: Sun, 24 Sep 2023 01:03:12 +0300 Subject: [PATCH 4/6] support sse for Anthropic --- src/components/chat-window.vue | 2 +- src/scripts/ts/modules/openAI.ts | 137 +++++++++---------------------- 2 files changed, 40 insertions(+), 99 deletions(-) diff --git a/src/components/chat-window.vue b/src/components/chat-window.vue index 78d0279..ca782b3 100644 --- a/src/components/chat-window.vue +++ b/src/components/chat-window.vue @@ -244,7 +244,7 @@ export default { this.messages.push(this.messages.pop()); // so it'll be stored properly in cache // newText = newMsg.content; } catch(err) { - let msg = err?.error?.message || err?.message; + let msg = err?.error?.message || err?.message || err; if (err.statusText) msg = `${err?.statusText} (${err?.statusCode})`; helpers.toast(`Failed to process request: ${msg || ''}`, 'is-danger', 'is-top'); diff --git a/src/scripts/ts/modules/openAI.ts b/src/scripts/ts/modules/openAI.ts index b104594..de2cd28 100644 --- a/src/scripts/ts/modules/openAI.ts +++ b/src/scripts/ts/modules/openAI.ts @@ -83,81 +83,6 @@ export class OpenAI { return json ?? res; } - // public async createChatCompletionStream(messages: IConvMessage[], config, priming?: string, onDelta?: (data) => void, onProgress?: (wholeData) => void) { - // const p = libx.newPromise(); - // const url = `https://api.openai.com/v1/chat/completions`; - // const apiKey = config.apikey; - // const payload = { - // "messages": [ - // { - // role: 'system', - // content: priming ?? `You are an assistant bot.`, - // }, - // ...messages - // ], - // ...{ - // // defaults: - // frequency_penalty: 0, - // presence_penalty: 0, - // temperature: 0.8, - // max_tokens: 256, // 512, - // model: "gpt-3.5-turbo", - // ...config, // override - // stream: true, - // // user: 'userId', - // top_p: 1.0, - // n: 1, - // } - // }; - // delete payload.apikey; - - // const eventSource = new SSE(url, { - // method: "POST", - // headers: { - // 'Accept': 'application/json, text/plain, */*', - // 'Authorization': `Bearer ${apiKey}`, - // 'Content-Type': 'application/json', - // }, - // payload: JSON.stringify({ - // ...payload, - // "stream": true, - // }), - // }) as SSE; - - // let contents = ''; - - // eventSource.addEventListener('error', (event: any) => { - // if (!contents) { - // libx.log.e('error: ', event, contents) - // p.reject(JSON.parse(event.data)); - // } - // }); - - // eventSource.addEventListener('message', async (event: any) => { - // if (event.data === '[DONE]') { - // libx.log.d('message: done: ', event, contents) - // p.resolve(contents); - // return; - // } - - // try { - // const chunk = this.parseChunk(event.data); - // if (chunk.choices && chunk.choices.length > 0) { - // const newData = chunk.choices[0]?.delta?.content || ''; - // contents += newData; - // if (onDelta) onDelta(newData); - // if (onProgress) onProgress(contents); - // } - // } catch (err) { - // console.error(err); - // p.reject(err); - // } - // }); - - // eventSource.stream(); - // return p; - // } - public async createChatCompletionStream({ url, @@ -181,8 +106,15 @@ export class OpenAI { eventSource.addEventListener("error", (event: any) => { if (!contents) { + let errorMessage; + try { + errorMessage = JSON.parse(event.data); + } catch (e) { + errorMessage = event.data; + } + libx.log.e("error: ", event, contents); - p.reject(event.data && JSON.parse(event.data)); + p.reject(errorMessage); } }); @@ -192,13 +124,11 @@ export class OpenAI { // Handle this special case, maybe close the event source or do some other logic libx.log.d("message: done: ", event, contents); p.resolve(contents); - // eventSource.close(); return; } try { - const chunk = this.parseChunk(event.data); - if (chunk.choices && chunk.choices.length > 0) { - const newData = getNewData(chunk); + const newData = getNewData(event.data); + if (newData) { contents += newData; if (onDelta) onDelta(newData); if (onProgress) onProgress(contents); @@ -230,7 +160,7 @@ export class OpenAI { } let httpResponse = await response.json(); let res = getNewData(httpResponse); - return onDelta(res); // Assuming httpResponse has a 'content' field + return onDelta(res); } public async createChatCompletionRes( @@ -242,8 +172,7 @@ export class OpenAI { ) { const p = libx.newPromise(); - //// payload - + //// payloads const setupPayloadForGPT = () => { return { messages: [ @@ -282,9 +211,8 @@ export class OpenAI { }; const setupPayloadForAnthropic = () => { - const promptContent = `\n\nHuman: ${messages - .map((message) => message.content) - .join("\n\nHuman: ")}\n\nAssistant:`; + const promptContent = `\n\nHuman: ${messages.length > 0 ? messages[messages.length - 1].content : "" + }\n\nAssistant:`; return { prompt: promptContent, model: "claude-2", // or "claude-instant-1" @@ -295,39 +223,49 @@ export class OpenAI { const setupPayloadForAI21 = () => { return { - // prompt: messages.map((message) => message.content).join("\n"), - prompt: messages.length > 0 ? messages[messages.length - 1].content : "", + prompt: + messages.length > 0 ? messages[messages.length - 1].content : "", model_type: config.model_type || "mid", // or 'light' or 'ultra' based on your choice maxTokens: 256, ...config, }; }; - /////////// handle responses - const handleGPTResponse = (chunk) => { - if (chunk.choices && chunk.choices.length > 0) { - return chunk.choices[0]?.delta?.content || ""; + //// handle responses + const handleGPTResponse = (data) => { + try { + const chunk = JSON.parse(data); + if (chunk.choices && chunk.choices.length > 0) { + return chunk.choices[0]?.delta?.content || ""; + } + } catch (err) { + console.error("Error parsing GPT response", err); } return ""; }; const handleCohereResponse = (chunk) => { - // For simplicity, I'm assuming you only request one generation. - // If you request multiple generations (using num_generations parameter > 1), - // you may want to handle all the returned texts appropriately. if (chunk.generations && chunk.generations.length > 0) { return chunk.generations[0].text; } throw new Error("No completion found in Cohere response."); }; - const handleAnthropicResponse = (chunk) => { - return chunk.completion; + + const handleAnthropicResponse = (data) => { + try { + const chunk = JSON.parse(data); + return chunk.completion || ""; + } catch (err) { + console.error("Error parsing Anthropics response", err); + return ""; + } }; const handleAI21Response = (httpResponse) => { return httpResponse?.completions[0]?.data?.text; }; + //get model type const getModelType = (model) => { if (model.includes("gpt") || model.includes("davinci")) { return "gpt"; @@ -349,6 +287,7 @@ export class OpenAI { "Content-Type": "application/json", }; let useSSE = false; + switch (modelType) { case "gpt": url = `https://api.openai.com/v1/chat/completions`; @@ -364,11 +303,13 @@ export class OpenAI { headers["Authorization"] = `Bearer ${config.apikey}`; break; case "anthropic": - url = `https://api.anthropic.com/v1/complete`; + // url = `https://api.anthropic.com/v1/complete`; + url = `http://localhost:3001/api`; payload = setupPayloadForAnthropic(); getNewData = (data) => handleAnthropicResponse(data); headers["x-api-key"] = config.apikey; headers["anthropic-version"] = "2023-06-01"; + useSSE = true; break; case "ai21": url = `https://api.ai21.com/studio/v1/j2-${config.model_type || "mid" From 041464cad57794f841cccc07cbb04b18f02c3dbd Mon Sep 17 00:00:00 2001 From: Gilad Dekel Date: Sat, 7 Oct 2023 21:07:04 -0500 Subject: [PATCH 5/6] improve --- src/components/chat-window.vue | 12 +- src/scripts/ts/modules/openAI.ts | 745 ++++++++++++++++--------------- 2 files changed, 386 insertions(+), 371 deletions(-) diff --git a/src/components/chat-window.vue b/src/components/chat-window.vue index ca782b3..004cbd7 100644 --- a/src/components/chat-window.vue +++ b/src/components/chat-window.vue @@ -231,12 +231,12 @@ export default { await this.altGpt.performSmartCompletion(messages, this.selectedPlugins, config, onDelta, priming, this.pluginsSettings); } else { // await this.openAI.createChatCompletionStream(messages, config, priming, onDelta); - const httpResponse = await this.openAI.createChatCompletionRes( - messages, - config, - priming, - onDelta - ); + await this.openAI.createChatCompletionRes( + messages, + config, + priming, + onDelta + ); } this.$forceUpdate(); diff --git a/src/scripts/ts/modules/openAI.ts b/src/scripts/ts/modules/openAI.ts index de2cd28..6bb7f2d 100644 --- a/src/scripts/ts/modules/openAI.ts +++ b/src/scripts/ts/modules/openAI.ts @@ -6,371 +6,386 @@ import { IConvMessage } from '../types/IConvMessage.js'; import { OpenAIResponseChunk } from '../types/OpenAIResponseChunk.js'; export class OpenAI { - public openaiLib: typeof openai; - public openaiApi; - - public constructor(public options?: Partial) { - this.options = { ...new ModuleOptions(), ...options }; - this.openaiLib = openai; - - const configuration = new openai.Configuration({ - apiKey: '', - }); - this.openaiApi = new openai.OpenAIApi(configuration); - } - - public async makeCompletion(prompt: string): Promise> { - const response = await this.openaiApi.createCompletion({ - model: 'text-davinci-003', // 'text-davinci-003', - prompt: prompt, - temperature: 0.1, - max_tokens: 512, - top_p: 1.0, - frequency_penalty: 0.0, - presence_penalty: 0.0, - // stop: [''], - }); - return response; - } - - public async createChatCompletion(messages: IConvMessage[], config, priming?: string) { - const p = libx.newPromise(); - const url = `https://api.openai.com/v1/chat/completions`; - const apiKey = config.apikey; - const payload = { - "messages": [ - { - role: 'system', - content: priming ?? `You are an assistant bot.`, - }, - ...messages - ], - ...{ - // defaults: - frequency_penalty: 0, - presence_penalty: 0, - temperature: 0.8, - max_tokens: 256, // 512, - model: "gpt-3.5-turbo", - ...config, // override - stream: false, - // user: 'userId', - top_p: 1.0, - n: 1, - } - }; - delete payload.apikey; - - const res: any = await fetch(url, { - method: "POST", - headers: { - 'Accept': 'application/json, text/plain, */*', - 'Authorization': `Bearer ${apiKey}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - ...payload, - "stream": false, - }), - }); - - const json = await res.json(); - - if (json.choices && json.choices.length > 0) { - return json.choices[0]?.message?.content || ''; - } - - return json ?? res; - } - - - public async createChatCompletionStream({ - url, - headers, - payload, - p, - getNewData, - onDelta, - onProgress, - }) { - const eventSource = new SSE(url, { - method: "POST", - headers: headers, - payload: JSON.stringify({ - ...payload, - stream: true, - }), - }) as SSE; - - let contents = ""; - - eventSource.addEventListener("error", (event: any) => { - if (!contents) { - let errorMessage; - try { - errorMessage = JSON.parse(event.data); - } catch (e) { - errorMessage = event.data; - } - - libx.log.e("error: ", event, contents); - p.reject(errorMessage); - } - }); - - - eventSource.addEventListener("message", async (event: any) => { - if (event?.data === "[DONE]") { - // Handle this special case, maybe close the event source or do some other logic - libx.log.d("message: done: ", event, contents); - p.resolve(contents); - return; - } - try { - const newData = getNewData(event.data); - if (newData) { - contents += newData; - if (onDelta) onDelta(newData); - if (onProgress) onProgress(contents); - } - } catch (err) { - console.error(err); - p.reject(err); - } - }); - eventSource.stream(); - - return p; - } - - public async createChatCompletionHTTP({ - url, - headers, - payload, - getNewData, - onDelta, - }) { - const response = await fetch(url, { - method: "POST", - headers: headers, - body: JSON.stringify(payload), - }); - if (!response.ok) { - throw new Error(`HTTP error! Status: ${response.status}`); - } - let httpResponse = await response.json(); - let res = getNewData(httpResponse); - return onDelta(res); - } - - public async createChatCompletionRes( - messages: IConvMessage[], - config, - priming?: string, - onDelta?: (data) => void, - onProgress?: (wholeData) => void - ) { - const p = libx.newPromise(); - - //// payloads - const setupPayloadForGPT = () => { - return { - messages: [ - { - role: "system", - content: priming ?? `You are an assistant bot.`, - }, - ...messages, - ], - ...{ - frequency_penalty: 0, - presence_penalty: 0, - temperature: 0.8, - max_tokens: 256, - model: "gpt-3.5-turbo", - ...config, - stream: true, - top_p: 1.0, - n: 1, - }, - }; - }; - - const setupPayloadForCohere = () => { - const promptContent = [ - priming ?? `You are an assistant bot.`, - ...messages.map((message) => message.content), - ].join("\n"); - return { - prompt: promptContent, - model: "command", - max_tokens: 256, - stream: true, - ...config, - }; - }; - - const setupPayloadForAnthropic = () => { - const promptContent = `\n\nHuman: ${messages.length > 0 ? messages[messages.length - 1].content : "" - }\n\nAssistant:`; - return { - prompt: promptContent, - model: "claude-2", // or "claude-instant-1" - max_tokens_to_sample: 300, // or another value you prefer - ...config, // to handle other configuration parameters like temperature, top_p, etc. - }; - }; - - const setupPayloadForAI21 = () => { - return { - prompt: - messages.length > 0 ? messages[messages.length - 1].content : "", - model_type: config.model_type || "mid", // or 'light' or 'ultra' based on your choice - maxTokens: 256, - ...config, - }; - }; - - //// handle responses - const handleGPTResponse = (data) => { - try { - const chunk = JSON.parse(data); - if (chunk.choices && chunk.choices.length > 0) { - return chunk.choices[0]?.delta?.content || ""; - } - } catch (err) { - console.error("Error parsing GPT response", err); - } - return ""; - }; - - const handleCohereResponse = (chunk) => { - if (chunk.generations && chunk.generations.length > 0) { - return chunk.generations[0].text; - } - throw new Error("No completion found in Cohere response."); - }; - - const handleAnthropicResponse = (data) => { - try { - const chunk = JSON.parse(data); - return chunk.completion || ""; - } catch (err) { - console.error("Error parsing Anthropics response", err); - return ""; - } - }; - - const handleAI21Response = (httpResponse) => { - return httpResponse?.completions[0]?.data?.text; - }; - - //get model type - const getModelType = (model) => { - if (model.includes("gpt") || model.includes("davinci")) { - return "gpt"; - } else if (model.includes("xlarge")) { - return "cohere"; - } else if (model.includes("claude")) { - return "anthropic"; - } else if (model.includes("j2")) { - return "ai21"; - } - }; - - let modelType = getModelType(config.model); - let url: string; - let payload: any; - let getNewData: (data: string) => string; - let headers = { - Accept: "application/json, text/plain, */*", - "Content-Type": "application/json", - }; - let useSSE = false; - - switch (modelType) { - case "gpt": - url = `https://api.openai.com/v1/chat/completions`; - payload = setupPayloadForGPT(); - getNewData = (data) => handleGPTResponse(data); - headers["Authorization"] = `Bearer ${config.apikey}`; - useSSE = true; - break; - case "cohere": - url = `https://api.cohere.ai/v1/generate`; - payload = setupPayloadForCohere(); - getNewData = (data) => handleCohereResponse(data); - headers["Authorization"] = `Bearer ${config.apikey}`; - break; - case "anthropic": - // url = `https://api.anthropic.com/v1/complete`; - url = `http://localhost:3001/api`; - payload = setupPayloadForAnthropic(); - getNewData = (data) => handleAnthropicResponse(data); - headers["x-api-key"] = config.apikey; - headers["anthropic-version"] = "2023-06-01"; - useSSE = true; - break; - case "ai21": - url = `https://api.ai21.com/studio/v1/j2-${config.model_type || "mid" - }/complete`; - payload = setupPayloadForAI21(); - getNewData = (data) => handleAI21Response(data); - headers["Authorization"] = `Bearer ${config.apikey}`; - break; - // Add more cases for other models in the future as needed - default: - throw new Error("Unsupported model type."); - } - - delete payload.apikey; - - console.log("Config:", config); - console.log("Messages:", messages); - - if (useSSE) { - return this.createChatCompletionStream({ - url, - headers, - payload, - p, - getNewData, - onDelta, - onProgress, - }); - } else { - return this.createChatCompletionHTTP({ - url, - headers, - payload, - getNewData, - onDelta, - }); - } - } - - private parseChunk(buffer: any): OpenAIResponseChunk { - const chunk = buffer.toString().split(/\s*data\:\s*/); - // for (let content of chunk) { - // content = content.trim(); - // } - // const chunk = buffer.toString().replace('data: ', '').trim(); - - if (libx.isEmptyString(chunk) || chunk === '[DONE]') { - return { - done: true, - }; - } - - const parsed = JSON.parse(chunk); - - return { - id: parsed.id, - done: false, - choices: parsed.choices, - model: parsed.model, - }; - } + public openaiLib: typeof openai; + public openaiApi; + + public constructor(public options?: Partial) { + this.options = { ...new ModuleOptions(), ...options }; + this.openaiLib = openai; + + const configuration = new openai.Configuration({ + apiKey: '', + }); + this.openaiApi = new openai.OpenAIApi(configuration); + } + + public async makeCompletion(prompt: string): Promise> { + const response = await this.openaiApi.createCompletion({ + model: 'text-davinci-003', // 'text-davinci-003', + prompt: prompt, + temperature: 0.1, + max_tokens: 512, + top_p: 1.0, + frequency_penalty: 0.0, + presence_penalty: 0.0, + // stop: [''], + }); + return response; + } + + public async createChatCompletion(messages: IConvMessage[], config, priming?: string) { + const p = libx.newPromise(); + const url = `https://api.openai.com/v1/chat/completions`; + const apiKey = config.apikey; + const payload = { + "messages": [ + { + role: 'system', + content: priming ?? `You are an assistant bot.`, + }, + ...messages + ], + ...{ + // defaults: + frequency_penalty: 0, + presence_penalty: 0, + temperature: 0.8, + max_tokens: 256, // 512, + model: "gpt-3.5-turbo", + ...config, // override + stream: false, + // user: 'userId', + top_p: 1.0, + n: 1, + } + }; + delete payload.apikey; + + const res: any = await fetch(url, { + method: "POST", + headers: { + 'Accept': 'application/json, text/plain, */*', + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + ...payload, + "stream": false, + }), + }); + + const json = await res.json(); + + if (json.choices && json.choices.length > 0) { + return json.choices[0]?.message?.content || ''; + } + + return json ?? res; + } + + public async createChatCompletionStream({ + url, + headers, + payload, + p, + getNewData, + onDelta, + onProgress, + }) { + const eventSource = new SSE(url, { + method: "POST", + headers: headers, + payload: JSON.stringify({ + ...payload, + stream: true, + }), + }) as SSE; + + let contents = ""; + + eventSource.addEventListener("error", (event: any) => { + if (!contents) { + let errorMessage; + try { + errorMessage = JSON.parse(event.data); + } catch (e) { + errorMessage = event.data; + } + + libx.log.e("error: ", event, contents); + p.reject(errorMessage); + } + }); + + eventSource.addEventListener("message", async (event: any) => { + if (event?.data === "[DONE]") { + // Handle this special case, maybe close the event source or do some other logic + libx.log.d("message: done: ", event, contents); + p.resolve(contents); + return; + } + try { + const newData = getNewData(event.data); + if (newData) { + contents += newData; + if (onDelta) onDelta(newData); + if (onProgress) onProgress(contents); + } + } catch (err) { + console.error(err); + p.reject(err); + } + }); + eventSource.stream(); + + return p; + } + + public async createChatCompletionHTTP({ + url, + headers, + payload, + getNewData, + onDelta, + }) { + const response = await fetch(url, { + method: "POST", + headers: headers, + body: JSON.stringify(payload), + }); + if (!response.ok) { + throw new Error(`HTTP error! Status: ${response.status}`); + } + let httpResponse = await response.json(); + let res = getNewData(httpResponse); + return onDelta(res); + } + + public async createChatCompletionRes( + messages: IConvMessage[], + config, + priming?: string, + onDelta?: (data) => void, + onProgress?: (wholeData) => void + ) { + const p = libx.newPromise(); + + //// payloads + const setupPayloadForGPT = (model) => { + return { + messages: [ + { + role: "system", + content: priming ?? `You are an assistant bot.`, + }, + ...messages, + ], + // messages: [ + // messages.length > 0 + // ? { role: "user", content: messages[messages.length - 1].content } + // : "", + // ], + ...{ + frequency_penalty: 0, + presence_penalty: 0, + temperature: 0.8, + max_tokens: 256, + model, + ...config, + stream: true, + top_p: 1.0, + n: 1, + }, + }; + }; + + const setupPayloadForDavinci = (model) => { + const userMessages = messages.filter((m) => m.role === "user"); + const prompt = + userMessages.length > 0 + ? userMessages[userMessages.length - 1].content + : ""; + return { + prompt, + frequency_penalty: 0, + presence_penalty: 0, + temperature: 0.8, + max_tokens: 256, + model, + top_p: 1.0, + n: 1, + ...config, + }; + }; + + const setupPayloadForCohere = (model) => { + return { + prompt: + messages.length > 0 ? messages[messages.length - 1].content : "", + model, + max_tokens: 256, + // stream: true, + ...config, + }; + }; + + const setupPayloadForAnthropic = (model) => { + const promptContent = `\n\nHuman: ${messages.length > 0 ? messages[messages.length - 1].content : "" + }\n\nAssistant:`; + return { + prompt: promptContent, + model, + max_tokens_to_sample: 300, + stream: true, + ...config, + }; + }; + + const setupPayloadForAI21 = (model) => { + return { + prompt: + messages.length > 0 ? messages[messages.length - 1].content : "", + model_type: model, + maxTokens: 256, + ...config, + }; + }; + + //// handle responses + const handleGPTResponse = (data) => { + try { + const chunk = JSON.parse(data); + if (chunk.choices && chunk.choices.length > 0) { + return chunk.choices[0]?.delta?.content || ""; + } + } catch (err) { + console.error("Error parsing GPT response", err); + } + return ""; + }; + + const handleDavinciResponse = (data) => { + return data.choices[0].text.trim(); + }; + + const handleCohereResponse = (chunk) => { + if (chunk.generations && chunk.generations.length > 0) { + return chunk.generations[0].text; + } + throw new Error("No completion found in Cohere response."); + }; + + const handleAnthropicResponse = (data) => { + try { + const chunk = JSON.parse(data); + return chunk.completion || ""; + } catch (err) { + console.error("Error parsing Anthropics response", err); + return ""; + } + }; + + const handleAI21Response = (httpResponse) => { + return httpResponse?.completions[0]?.data?.text; + }; + + let url: string; + let payload: any; + let getNewData: (data: string) => string; + let headers = { + Accept: "application/json, text/plain, */*", + "Content-Type": "application/json", + }; + + switch (config.model) { + case "gpt-3.5-turbo-0301": + case "gpt-4-0314": + url = `https://api.openai.com/v1/chat/completions`; + payload = setupPayloadForGPT(config.model); + getNewData = (data) => handleGPTResponse(data); + headers["Authorization"] = `Bearer ${config.apikey}`; + break; + case "text-davinci-002": + url = "https://api.openai.com/v1/completions"; + payload = setupPayloadForDavinci(config.model); + getNewData = (data) => handleDavinciResponse(data); + headers["Authorization"] = `Bearer ${config.apikey}`; + break; + case "command-xlarge-nightly": + case "xlarge": + url = `https://api.cohere.ai/v1/generate`; + payload = setupPayloadForCohere(config.model); + getNewData = (data) => handleCohereResponse(data); + headers["Authorization"] = `Bearer ${config.apikey}`; + break; + case "claude-instant-v1.0": + // url = `https://api.anthropic.com/v1/complete`; + url = `http://localhost:3001/api`; + payload = setupPayloadForAnthropic(config.model); + getNewData = (data) => handleAnthropicResponse(data); + headers["x-api-key"] = config.apikey; + headers["anthropic-version"] = "2023-06-01"; + break; + case "j2-grande-instruct": + url = `https://api.ai21.com/studio/v1/j2-${config.model_type || "mid" + }/complete`; + payload = setupPayloadForAI21(config.model); + getNewData = (data) => handleAI21Response(data); + headers["Authorization"] = `Bearer ${config.apikey}`; + break; + // Add more cases for other models in the future as needed + default: + throw new Error("Unsupported model type."); + } + + delete payload.apikey; + + console.log("Config:", config); + console.log("Messages:", messages); + + if (payload.stream) { + return this.createChatCompletionStream({ + url, + headers, + payload, + p, + getNewData, + onDelta, + onProgress, + }); + } else { + return this.createChatCompletionHTTP({ + url, + headers, + payload, + getNewData, + onDelta, + }); + } + } + + private parseChunk(buffer: any): OpenAIResponseChunk { + const chunk = buffer.toString().split(/\s*data\:\s*/); + // for (let content of chunk) { + // content = content.trim(); + // } + // const chunk = buffer.toString().replace('data: ', '').trim(); + + if (libx.isEmptyString(chunk) || chunk === "[DONE]") { + return { + done: true, + }; + } + + const parsed = JSON.parse(chunk); + + return { + id: parsed.id, + done: false, + choices: parsed.choices, + model: parsed.model, + }; + } } export class ModuleOptions { } From d242172d07eb65d63fee3e9a290539b8e19b9bc2 Mon Sep 17 00:00:00 2001 From: Gilad Dekel Date: Fri, 5 Jan 2024 12:23:33 +0200 Subject: [PATCH 6/6] change the endpoint for Anthropic --- src/scripts/ts/modules/openAI.ts | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/src/scripts/ts/modules/openAI.ts b/src/scripts/ts/modules/openAI.ts index 6bb7f2d..37f1446 100644 --- a/src/scripts/ts/modules/openAI.ts +++ b/src/scripts/ts/modules/openAI.ts @@ -118,12 +118,19 @@ export class OpenAI { }); eventSource.addEventListener("message", async (event: any) => { + console.log("🚀 ~ file: openAI.ts:140 ~ OpenAI ~ eventSource.addEventListener ~ event:", event); if (event?.data === "[DONE]") { // Handle this special case, maybe close the event source or do some other logic libx.log.d("message: done: ", event, contents); p.resolve(contents); return; } + + if (!event.data || event.data.trim() === "") { + console.log("Received empty or invalid data."); + return; // Skip processing for empty data + } + try { const newData = getNewData(event.data); if (newData) { @@ -136,6 +143,7 @@ export class OpenAI { p.reject(err); } }); + eventSource.stream(); return p; @@ -230,17 +238,25 @@ export class OpenAI { }; const setupPayloadForAnthropic = (model) => { - const promptContent = `\n\nHuman: ${messages.length > 0 ? messages[messages.length - 1].content : "" + const systemPrompt = `\n\nHuman: ${messages.length > 0 ? messages[messages.length - 1].content : "" }\n\nAssistant:`; + const newConfig = { + provider: "anthropic", + model: model, + maxTokens: 300, + stream: true, + frequencyPenalty: config.frequencyPenalty || 0, + presencePenalty: config.presencePenalty || 0, + temperature: config.temperature || 0.8, + }; + return { - prompt: promptContent, - model, - max_tokens_to_sample: 300, + systemPrompt, + config: newConfig, stream: true, - ...config, + }; }; - const setupPayloadForAI21 = (model) => { return { prompt: @@ -320,7 +336,8 @@ export class OpenAI { break; case "claude-instant-v1.0": // url = `https://api.anthropic.com/v1/complete`; - url = `http://localhost:3001/api`; + // url = `http://localhost:3001/api`; + url = `https://ws-edge-v2.feedox.workers.dev/completion/650fb33a815c45d4191a1e094628ec7d/` payload = setupPayloadForAnthropic(config.model); getNewData = (data) => handleAnthropicResponse(data); headers["x-api-key"] = config.apikey;