From b1a1dcebf8b324f3e192f1345d162a19c7143ddc Mon Sep 17 00:00:00 2001 From: Shawn Esquivel Date: Sat, 10 Jun 2023 11:13:41 -0700 Subject: [PATCH 01/17] fill in solutions --- agents/ResearchAgent-soln.js | 81 -------- agents/ResearchAgent.js | 62 +++++- app/memory/memory-solution.jsx | 100 ---------- app/memory/page.jsx | 101 +++++++++- app/page-template/page.jsx | 76 ------- app/streaming/page.jsx | 46 ++++- app/streaming/streaming-solution.jsx | 102 ---------- pages/api/chatcompletions.js | 22 ++- pages/api/content-generator.js | 110 ++++++++++- pages/api/memory.js | 37 ++++ pages/api/pdf-query.js | 28 ++- pages/api/pdf-upload.js | 107 +++++++++- pages/api/resume-query-metadata.js | 48 +++++ pages/api/resume-upload.js | 97 ++++++++- pages/api/solutions/chatcompletions-soln.js | 33 ---- pages/api/solutions/content-generator-soln.js | 187 ------------------ pages/api/solutions/memory-soln.js | 37 ---- pages/api/solutions/pdf-query-soln.js | 58 ------ pages/api/solutions/pdf-upload-soln.js | 123 ------------ .../solutions/resume-query-metadata-soln.js | 75 ------- pages/api/solutions/resume-upload-soln.js | 129 ------------ pages/api/solutions/streaming-soln.js | 46 ----- pages/api/solutions/video-chat-soln.js | 187 ------------------ pages/api/streaming.js | 23 ++- pages/api/video-chat.js | 144 +++++++++++++- tools/SerpAPI.js | 19 +- tools/WebBrowser.js | 13 +- tools/solutions/SerpAPI-soln.js | 21 -- tools/solutions/WebBrowser-soln.js | 20 -- 29 files changed, 821 insertions(+), 1311 deletions(-) delete mode 100644 agents/ResearchAgent-soln.js delete mode 100644 app/memory/memory-solution.jsx delete mode 100644 app/page-template/page.jsx delete mode 100644 app/streaming/streaming-solution.jsx delete mode 100644 pages/api/solutions/chatcompletions-soln.js delete mode 100644 pages/api/solutions/content-generator-soln.js delete mode 100644 pages/api/solutions/memory-soln.js delete mode 100644 pages/api/solutions/pdf-query-soln.js delete mode 100644 pages/api/solutions/pdf-upload-soln.js delete mode 100644 pages/api/solutions/resume-query-metadata-soln.js delete mode 100644 pages/api/solutions/resume-upload-soln.js delete mode 100644 pages/api/solutions/streaming-soln.js delete mode 100644 pages/api/solutions/video-chat-soln.js delete mode 100644 tools/solutions/SerpAPI-soln.js delete mode 100644 tools/solutions/WebBrowser-soln.js diff --git a/agents/ResearchAgent-soln.js b/agents/ResearchAgent-soln.js deleted file mode 100644 index b7fbf06..0000000 --- a/agents/ResearchAgent-soln.js +++ /dev/null @@ -1,81 +0,0 @@ -import { ChatOpenAI } from "langchain/chat_models/openai"; -import { LLMChain } from "langchain/chains"; -import { ZeroShotAgent } from "langchain/agents"; -import { - ChatPromptTemplate, - HumanMessagePromptTemplate, - SystemMessagePromptTemplate, -} from "langchain/prompts"; -import { AgentExecutor } from "langchain/agents"; -import SerpAPITool from "../tools/SerpAPI"; -import WebBrowserTool from "../tools/WebBrowser"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ -const ResearchAgent = async (topic) => { - console.log({ topic }); - try { - // We'll give it tohe ability to search Google and also Browse the WEb - // show the importance of returnDirect - const SerpAPI = SerpAPITool(); - const WebBrowser = WebBrowserTool(); - - console.log(SerpAPI.returnDirect); - console.log(WebBrowser.returnDirect); - // We put them into an array of tools - const tools = [SerpAPI, WebBrowser]; - - // We'll use the ZeroShotReactDescription which is the recommended tool for chat models - // https://js.langchain.com/docs/modules/agents/agents/ - // The cool thing is that Langchain gives us a method that can create prompts specifically for that agent - const promptTemplate = ZeroShotAgent.createPrompt(tools, { - prefix: `Answer the following questions as best you can. You have access to the following tools:`, - suffix: `Begin! Answer concisely. It's OK to say you don't know.`, - }); - - // Then we'll initialize it with a Prompt template again - const chatPrompt = ChatPromptTemplate.fromPromptMessages([ - new SystemMessagePromptTemplate(promptTemplate), - HumanMessagePromptTemplate.fromTemplate(`{input}`), - ]); - - // And we'll initialize the model, what is chatOpenAI since we're using a ChatAgent - const chat = new ChatOpenAI({}); - // We'll create an LLM chain which just a prompt template and a LLM or chat model - const llmChain = new LLMChain({ - prompt: chatPrompt, - llm: chat, - }); - // then we'lll use that LLM chain as the basis of the agent - // so basidcally our agent is made up of: Tools, LLM and Prompt Templates, making it highly customizable to our needs! - const agent = new ZeroShotAgent({ - llmChain, - allowedTools: tools.map((tool) => tool.name), - }); - // Now we'll create an Executor instance which allows us to make queries to the agent - - const executor = AgentExecutor.fromAgentAndTools({ - agent, - tools, - returnIntermediateSteps: false, - // Max iterations is important - because sometimes our agent can get confused. this can be from a variety of factors such as: - // not using the right agent for the tool - // prompts are not perfect – check out the document for crafting the perfect prompt! - // always set this to a low number to start, or if you're not going to watch the output - maxIterations: 3, - // Always set verbose to true, there was a case where I didn't do this and realized my agent was going in loops - verbose: true, - }); - - const result = await executor.run(`Who is ${topic}?`); - - return result; - } catch (err) { - console.error(err); - return "Error in completing research"; - } -}; -export default ResearchAgent; diff --git a/agents/ResearchAgent.js b/agents/ResearchAgent.js index ae1dda9..b7fbf06 100644 --- a/agents/ResearchAgent.js +++ b/agents/ResearchAgent.js @@ -10,16 +10,72 @@ import { AgentExecutor } from "langchain/agents"; import SerpAPITool from "../tools/SerpAPI"; import WebBrowserTool from "../tools/WebBrowser"; +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ const ResearchAgent = async (topic) => { console.log({ topic }); - try { - // do stuff + // We'll give it tohe ability to search Google and also Browse the WEb + // show the importance of returnDirect + const SerpAPI = SerpAPITool(); + const WebBrowser = WebBrowserTool(); + + console.log(SerpAPI.returnDirect); + console.log(WebBrowser.returnDirect); + // We put them into an array of tools + const tools = [SerpAPI, WebBrowser]; + + // We'll use the ZeroShotReactDescription which is the recommended tool for chat models + // https://js.langchain.com/docs/modules/agents/agents/ + // The cool thing is that Langchain gives us a method that can create prompts specifically for that agent + const promptTemplate = ZeroShotAgent.createPrompt(tools, { + prefix: `Answer the following questions as best you can. You have access to the following tools:`, + suffix: `Begin! Answer concisely. It's OK to say you don't know.`, + }); + + // Then we'll initialize it with a Prompt template again + const chatPrompt = ChatPromptTemplate.fromPromptMessages([ + new SystemMessagePromptTemplate(promptTemplate), + HumanMessagePromptTemplate.fromTemplate(`{input}`), + ]); + + // And we'll initialize the model, what is chatOpenAI since we're using a ChatAgent + const chat = new ChatOpenAI({}); + // We'll create an LLM chain which just a prompt template and a LLM or chat model + const llmChain = new LLMChain({ + prompt: chatPrompt, + llm: chat, + }); + // then we'lll use that LLM chain as the basis of the agent + // so basidcally our agent is made up of: Tools, LLM and Prompt Templates, making it highly customizable to our needs! + const agent = new ZeroShotAgent({ + llmChain, + allowedTools: tools.map((tool) => tool.name), + }); + // Now we'll create an Executor instance which allows us to make queries to the agent + + const executor = AgentExecutor.fromAgentAndTools({ + agent, + tools, + returnIntermediateSteps: false, + // Max iterations is important - because sometimes our agent can get confused. this can be from a variety of factors such as: + // not using the right agent for the tool + // prompts are not perfect – check out the document for crafting the perfect prompt! + // always set this to a low number to start, or if you're not going to watch the output + maxIterations: 3, + // Always set verbose to true, there was a case where I didn't do this and realized my agent was going in loops + verbose: true, + }); + + const result = await executor.run(`Who is ${topic}?`); return result; } catch (err) { console.error(err); + return "Error in completing research"; } }; - export default ResearchAgent; diff --git a/app/memory/memory-solution.jsx b/app/memory/memory-solution.jsx deleted file mode 100644 index f74e526..0000000 --- a/app/memory/memory-solution.jsx +++ /dev/null @@ -1,100 +0,0 @@ -"use client"; -import React, { useState } from "react"; -import PageHeader from "../components/PageHeader"; -import PromptBox from "../components/PromptBox"; -import Title from "../components/Title"; -import TwoColumnLayout from "../components/TwoColumnLayout"; -import ResultWithSources from "../components/ResultWithSources"; -import "../globals.css"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ - -const Memory = () => { - const [prompt, setPrompt] = useState(""); - const [error, setError] = useState(null); - const [messages, setMessages] = useState([ - { - text: "Hi there! What's your name and favourite food?", - type: "bot", - }, - ]); - const [firstMsg, setFirstMsg] = useState(true); - - const handlePromptChange = (e) => { - setPrompt(e.target.value); - }; - - const handleSubmitPrompt = async () => { - console.log("sending ", prompt); - try { - // Update the user message - setMessages((prevMessages) => [ - ...prevMessages, - { text: prompt, type: "user", sourceDocuments: null }, - ]); - - const response = await fetch("/api/memory", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ input: prompt, firstMsg }), - }); - - if (!response.ok) { - throw new Error(`HTTP Error! Status: ${response.status}`); - } - - setPrompt(""); - // So we don't reinitialize the chain - setFirstMsg(false); - const searchRes = await response.json(); - // Add the bot message - setMessages((prevMessages) => [ - ...prevMessages, - { text: searchRes.output.response, type: "bot", sourceDocuments: null }, - ]); - - console.log({ searchRes }); - // Clear any old error messages - setError(""); - } catch (err) { - console.error(err); - setError(err); - } - }; - - return ( - <> - - <TwoColumnLayout - leftChildren={ - <> - <PageHeader - heading="I remember everything" - boldText="Let's see if it can remember your name and favourite food. This tool will let you ask anything contained in a PDF document. " - description="This tool uses Buffer Memory and Conversation Chain. Head over to Module X to get started!" - /> - </> - } - rightChildren={ - <> - <ResultWithSources messages={messages} pngFile="brain" /> - <PromptBox - prompt={prompt} - handleSubmit={handleSubmitPrompt} - error={error} - handlePromptChange={handlePromptChange} - /> - </> - } - /> - </> - ); -}; - -export default Memory; diff --git a/app/memory/page.jsx b/app/memory/page.jsx index 541e4f2..f74e526 100644 --- a/app/memory/page.jsx +++ b/app/memory/page.jsx @@ -1 +1,100 @@ -// start here +"use client"; +import React, { useState } from "react"; +import PageHeader from "../components/PageHeader"; +import PromptBox from "../components/PromptBox"; +import Title from "../components/Title"; +import TwoColumnLayout from "../components/TwoColumnLayout"; +import ResultWithSources from "../components/ResultWithSources"; +import "../globals.css"; + +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ + +const Memory = () => { + const [prompt, setPrompt] = useState(""); + const [error, setError] = useState(null); + const [messages, setMessages] = useState([ + { + text: "Hi there! What's your name and favourite food?", + type: "bot", + }, + ]); + const [firstMsg, setFirstMsg] = useState(true); + + const handlePromptChange = (e) => { + setPrompt(e.target.value); + }; + + const handleSubmitPrompt = async () => { + console.log("sending ", prompt); + try { + // Update the user message + setMessages((prevMessages) => [ + ...prevMessages, + { text: prompt, type: "user", sourceDocuments: null }, + ]); + + const response = await fetch("/api/memory", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ input: prompt, firstMsg }), + }); + + if (!response.ok) { + throw new Error(`HTTP Error! Status: ${response.status}`); + } + + setPrompt(""); + // So we don't reinitialize the chain + setFirstMsg(false); + const searchRes = await response.json(); + // Add the bot message + setMessages((prevMessages) => [ + ...prevMessages, + { text: searchRes.output.response, type: "bot", sourceDocuments: null }, + ]); + + console.log({ searchRes }); + // Clear any old error messages + setError(""); + } catch (err) { + console.error(err); + setError(err); + } + }; + + return ( + <> + <Title headingText={"Memory"} emoji="🧠" /> + <TwoColumnLayout + leftChildren={ + <> + <PageHeader + heading="I remember everything" + boldText="Let's see if it can remember your name and favourite food. This tool will let you ask anything contained in a PDF document. " + description="This tool uses Buffer Memory and Conversation Chain. Head over to Module X to get started!" + /> + </> + } + rightChildren={ + <> + <ResultWithSources messages={messages} pngFile="brain" /> + <PromptBox + prompt={prompt} + handleSubmit={handleSubmitPrompt} + error={error} + handlePromptChange={handlePromptChange} + /> + </> + } + /> + </> + ); +}; + +export default Memory; diff --git a/app/page-template/page.jsx b/app/page-template/page.jsx deleted file mode 100644 index 215d9a1..0000000 --- a/app/page-template/page.jsx +++ /dev/null @@ -1,76 +0,0 @@ -"use client"; - -import React, { useState } from "react"; -import ResultWithSources from "../components/ResultWithSources"; -import PromptBox from "../components/PromptBox"; -import Button from "../components/Button"; -import PageHeader from "../components/PageHeader"; -import Title from "../components/Title"; -import TwoColumnLayout from "../components/TwoColumnLayout"; -import ButtonContainer from "../components/ButtonContainer"; -import "../globals.css"; - -const AnyComponentName = () => { - const [prompt, setPrompt] = useState(""); - const [data, setData] = useState(""); - - const handlePromptChange = (e) => { - setPrompt(e.target.value); - }; - - const handleSubmit = async () => { - console.log(`sending ${prompt}`); - // STEP 1: Modify Endpoint - const response = await fetch("/api/", { - // STEP 2: Check Method - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ input: prompt }), - }); - - const searchRes = await response.json(); - // Step 3: Double check the console log and setData accordingly - console.log(searchRes); - setData(searchRes.output); - }; - - return ( - <> - <Title emoji="💬" headingText="PDF-GPT" /> - <TwoColumnLayout - leftChildren={ - <> - <PageHeader - heading="Catch Title" - boldText="Bold Text" - description="Description" - /> - <ButtonContainer> - <Button - handleSubmit={handleSubmit} - endpoint="pdfupload-book" - buttonText="Upload" - /> - </ButtonContainer> - </> - } - rightChildren={ - <> - <ResultWithSources messages={messages} /> - <PromptBox - prompt={prompt} - handlePromptChange={handlePromptChange} - handleSubmit={() => handleSubmitQuery("/endpoint")} - placeHolderText={"...."} - error={error} - /> - </> - } - /> - </> - ); -}; - -export default AnyComponentName; diff --git a/app/streaming/page.jsx b/app/streaming/page.jsx index 1997075..900fa22 100644 --- a/app/streaming/page.jsx +++ b/app/streaming/page.jsx @@ -6,15 +6,19 @@ import ResultStreaming from "../components/ResultStreaming"; import Title from "../components/Title"; import TwoColumnLayout from "app/components/TwoColumnLayout"; +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ const Streaming = () => { const [prompt, setPrompt] = useState(""); const [error, setError] = useState(null); const [data, setData] = useState(""); - // add code + const [source, setSource] = useState(null); const processToken = (token) => { - // add code - return; + return token.replace(/\\n/g, "\n").replace(/\"/g, ""); }; const handlePromptChange = (e) => { @@ -23,7 +27,32 @@ const Streaming = () => { const handleSubmit = async () => { try { - // add code + console.log(`sending ${prompt}`); + await fetch("/api/streaming", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ input: prompt }), + }); + // close existing sources + if (source) { + source.close(); + } + // create new eventsource + + const newSource = new EventSource("/api/streaming"); + + setSource(newSource); + + newSource.addEventListener("newToken", (event) => { + const token = processToken(event.data); + setData((prevData) => prevData + token); + }); + + newSource.addEventListener("end", () => { + newSource.close(); + }); } catch (err) { console.error(err); setError(error); @@ -31,7 +60,14 @@ const Streaming = () => { }; // Clean up the EventSource on component unmount - // add code + useEffect(() => { + // stuff is gonna happen + return () => { + if (source) { + source.close(); + } + }; + }, [source]); return ( <> <Title emoji="💭" headingText="Streaming" /> diff --git a/app/streaming/streaming-solution.jsx b/app/streaming/streaming-solution.jsx deleted file mode 100644 index 900fa22..0000000 --- a/app/streaming/streaming-solution.jsx +++ /dev/null @@ -1,102 +0,0 @@ -"use client"; -import React, { useState, useEffect } from "react"; -import PageHeader from "../components/PageHeader"; -import PromptBox from "../components/PromptBox"; -import ResultStreaming from "../components/ResultStreaming"; -import Title from "../components/Title"; -import TwoColumnLayout from "app/components/TwoColumnLayout"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ -const Streaming = () => { - const [prompt, setPrompt] = useState(""); - const [error, setError] = useState(null); - const [data, setData] = useState(""); - const [source, setSource] = useState(null); - - const processToken = (token) => { - return token.replace(/\\n/g, "\n").replace(/\"/g, ""); - }; - - const handlePromptChange = (e) => { - setPrompt(e.target.value); - }; - - const handleSubmit = async () => { - try { - console.log(`sending ${prompt}`); - await fetch("/api/streaming", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ input: prompt }), - }); - // close existing sources - if (source) { - source.close(); - } - // create new eventsource - - const newSource = new EventSource("/api/streaming"); - - setSource(newSource); - - newSource.addEventListener("newToken", (event) => { - const token = processToken(event.data); - setData((prevData) => prevData + token); - }); - - newSource.addEventListener("end", () => { - newSource.close(); - }); - } catch (err) { - console.error(err); - setError(error); - } - }; - - // Clean up the EventSource on component unmount - useEffect(() => { - // stuff is gonna happen - return () => { - if (source) { - source.close(); - } - }; - }, [source]); - return ( - <> - <Title emoji="💭" headingText="Streaming" /> - <TwoColumnLayout - leftChildren={ - <> - <PageHeader - heading="Spit a Rap." - boldText="Nobody likes waiting for APIs to load. Use streaming to improve the user experience of chat bots." - description="This tutorial uses streaming. Head over to Module X to get started!" - /> - </> - } - rightChildren={ - <> - <ResultStreaming data={data} /> - <PromptBox - prompt={prompt} - handlePromptChange={handlePromptChange} - handleSubmit={handleSubmit} - placeHolderText={"Enter your name and city"} - error={error} - pngFile="pdf" - /> - </> - } - /> - </> - ); -}; - -export default Streaming; diff --git a/pages/api/chatcompletions.js b/pages/api/chatcompletions.js index 4c6c200..0d9e7c0 100644 --- a/pages/api/chatcompletions.js +++ b/pages/api/chatcompletions.js @@ -1,15 +1,29 @@ import { ChatOpenAI } from "langchain/chat_models/openai"; -import { HumanChatMessage } from "langchain/schema"; +import { HumanChatMessage, SystemChatMessage } from "langchain/schema"; -// create instance of chatOpenAI +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ + +const chat = new ChatOpenAI({ temperature: 0, modelName: "gpt-3.5-turbo" }); export default async function handler(req, res) { if (req.method === "POST") { // Grab the user prompt - // console.log(process.env.OPENAI_API_KEY); - // console.log(process.env.SERPAPI_API_KEY); + const { input } = req.body; + + if (!input) { + throw new Error("No input"); + } // Enter your code here + const response = await chat.call([ + new HumanChatMessage(`How do I write a for loop in ${input}?`), + ]); + + console.log(response); // Modify output as needed return res.status(200).json({ result: response }); diff --git a/pages/api/content-generator.js b/pages/api/content-generator.js index 47416c4..d5d7648 100644 --- a/pages/api/content-generator.js +++ b/pages/api/content-generator.js @@ -1,22 +1,74 @@ // /pages/api/transcript_chat.js + import { YoutubeTranscript } from "youtube-transcript"; +import extractVideoId from "../../utils/extractVideoId"; +import getVideoMetaData from "../../utils/getVideoMetaData"; import { ChatOpenAI } from "langchain/chat_models/openai"; import { LLMChain } from "langchain/chains"; +import ResearchAgent from "../../agents/ResearchAgent"; import { ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate, } from "langchain/prompts"; -import extractVideoId from "../../utils/extractVideoId"; -import getVideoMetaData from "../../utils/getVideoMetaData"; -import ResearchAgent from "../../agents/ResearchAgent"; + +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ // Global Variables +let chain; +let chatHistory = []; +let transcript = ""; +let metadataString = ""; +let research; // Initialize Chain with Data const initChain = async (transcript, metadataString, research, topic) => { try { - // do stuff + // For chat models, we provide a `ChatPromptTemplate` class that can be used to format chat prompts. + const llm = new ChatOpenAI({ + temperature: 0.7, + modelName: "gpt-3.5-turbo", + }); + + console.log(`Initializing Chat Prompt`); + + // For chat models, we provide a `ChatPromptTemplate` class that can be used to format chat prompts. + // This allows us to set the template that the bot sees every time + const chatPrompt = ChatPromptTemplate.fromPromptMessages([ + SystemMessagePromptTemplate.fromTemplate( + "You are a helpful social media assistant that provides research, new content, and advice to me. \n You are given the transcript of the video: {transcript} \n and video metadata: {metadata} as well as additional research: {research}" + ), + HumanMessagePromptTemplate.fromTemplate( + "{input}. Remember to use the video transcript and research as reference." + ), + ]); + + const question = `Write me a script for a new video that provides commentary on this video in a lighthearted, joking manner. It should compliment ${topic} with puns.`; + console.log(question); + + chain = new LLMChain({ + prompt: chatPrompt, + llm: llm, + // memory, + }); + + const response = await chain.call({ + transcript, + metadata: metadataString, + research, + input: question, + }); + + console.log({ response }); + + chatHistory.push({ + role: "assistant", + content: response.text, + }); return response; } catch (error) { @@ -50,7 +102,38 @@ export default async function handler(req, res) { if (firstMsg) { console.log("Received URL"); try { - // Initialize chain with transcript, metadata, research, and topic + const videoId = extractVideoId(prompt); + // API call for video transcript (same as last video, but we just grab the array and flatten it into a variable)[{text:" "},{ text: ""}] + const transcriptResponse = await YoutubeTranscript.fetchTranscript( + prompt + ); + transcriptResponse.forEach((line) => { + transcript += line.text; + }); + // Some error handling + if (!transcriptResponse) { + return res.status(400).json({ error: "Failed to get transcript" }); + } + + // API call for video metadata –– go to VideoMetaData and explain this + const metadata = await getVideoMetaData(videoId); + + // JSON object { [], [], [] } , null (no characters between), and use 2 spaces for indentation + metadataString = JSON.stringify(metadata, null, 2); + console.log({ metadataString }); + + // ResearchAgent + research = await ResearchAgent(topic); + + console.log({ research }); + + // Alright, finally we have all the context and we can initialize the chain! + const response = await initChain( + transcript, + metadataString, + research, + topic + ); // return res.status(200).json({ output: research }); return res.status(200).json({ @@ -70,8 +153,23 @@ export default async function handler(req, res) { // Very similar to previous section, don't worry too much about this just copy and paste it from the previous section! console.log("Received question"); try { - // do stuff + const question = prompt; + + console.log("Asking:", question); + console.log("Using old chain:", chain); + // Everytime we call the chain we need to pass all the context back so that it can fill in the prompt template appropriately + const response = await chain.call({ + transcript, + metadata: metadataString, + research, + input: question, + }); + // update chat history + chatHistory.push({ + role: "assistant", + content: response.text, + }); // just make sure to modify this response as necessary. return res.status(200).json({ output: response, diff --git a/pages/api/memory.js b/pages/api/memory.js index e69de29..8a806ed 100644 --- a/pages/api/memory.js +++ b/pages/api/memory.js @@ -0,0 +1,37 @@ +// Solution +import { OpenAI } from "langchain/llms/openai"; +import { BufferMemory } from "langchain/memory"; +import { ConversationChain } from "langchain/chains"; + +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ + +let model; +let memory; +let chain; +export default async function handler(req, res) { + if (req.method === "POST") { + const { input, firstMsg } = req.body; + + if (!input) { + throw new Error("No input!"); + } + + if (firstMsg) { + console.log("initializing chain"); + model = new OpenAI({ modelName: "gpt-3.5-turbo" }); + memory = new BufferMemory(); + chain = new ConversationChain({ llm: model, memory: memory }); + } + + console.log({ input }); + const response = await chain.call({ input }); + console.log({ response }); + return res.status(200).json({ output: response }); + } else { + res.status(405).json({ message: "Only POST is allowed" }); + } +} diff --git a/pages/api/pdf-query.js b/pages/api/pdf-query.js index 04117c1..bb345c7 100644 --- a/pages/api/pdf-query.js +++ b/pages/api/pdf-query.js @@ -3,6 +3,11 @@ import { VectorDBQAChain } from "langchain/chains"; import { OpenAIEmbeddings } from "langchain/embeddings/openai"; import { OpenAI } from "langchain/llms/openai"; import { PineconeStore } from "langchain/vectorstores/pinecone"; +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ // Example: https://js.langchain.com/docs/modules/indexes/document_loaders/examples/file_loaders/pdf export default async function handler(req, res) { @@ -22,11 +27,28 @@ export default async function handler(req, res) { console.log("input received:", input); - /* Use as part of a chain (currently no metadata filters) */ + const client = new PineconeClient(); + await client.init({ + apiKey: process.env.PINECONE_API_KEY, + environment: process.env.PINECONE_ENVIRONMENT, + }); + const pineconeIndex = client.Index(process.env.PINECONE_INDEX); - // Initialize Pinecone + const vectorStore = await PineconeStore.fromExistingIndex( + new OpenAIEmbeddings(), + { pineconeIndex } + ); - // Search! + /* Part Two: Use as part of a chain (currently no metadata filters) */ + + const model = new OpenAI(); + const chain = VectorDBQAChain.fromLLM(model, vectorStore, { + k: 1, + returnSourceDocuments: true, + }); + const response = await chain.call({ query: input }); + + console.log(response); return res.status(200).json({ result: response }); } catch (error) { diff --git a/pages/api/pdf-upload.js b/pages/api/pdf-upload.js index 5286fe5..2b808d4 100644 --- a/pages/api/pdf-upload.js +++ b/pages/api/pdf-upload.js @@ -5,21 +5,118 @@ import { OpenAIEmbeddings } from "langchain/embeddings/openai"; import { PineconeStore } from "langchain/vectorstores/pinecone"; import { CharacterTextSplitter } from "langchain/text_splitter"; +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ + // Example: https://js.langchain.com/docs/modules/indexes/document_loaders/examples/file_loaders/pdf + +/** + * + * INSTRUCTIONS + * 1. Run with book + error { + name: 'PineconeError', + source: 'server', + message: 'PineconeClient: Error calling upsert: PineconeError: metadata size is 140052 bytes, which exceeds the limit of 40960 bytes per vector', + stack: '' + } + * 2. Explain why -- vector meta data sizes is too big. + Language Models are often limited by the amount of text that you can pass to them. Therefore, it is neccessary to split them up into smaller chunks. LangChain provides several utilities for doing so. + https://js.langchain.com/docs/modules/indexes/text_splitters/ + + Play with chunk sizes... too small and you can't understand. + Fine tune this to your liking. + More vectors = more $$ + + + 3. Pinecone size 1536 + https://platform.openai.com/docs/guides/embeddings/second-generation-models + + 4. Upsert metadata size -- add this after split Docs + + // Reduce the size of the metadata for each document + const reducedDocs = splitDocs.map(doc => { + const reducedMetadata = { ...doc.metadata }; + delete reducedMetadata.pdf; // Remove the 'pdf' field + return new Document({ + pageContent: doc.pageContent, + metadata: reducedMetadata, + }); +}); + + + + + + * */ + export default async function handler(req, res) { if (req.method === "GET") { - console.log("Inside the PDF handler"); + console.log("Uploading book"); // Enter your code here /** STEP ONE: LOAD DOCUMENT */ + const bookPath = + "/Users/shawnesquivel/GitHub/yt-script-generator/data/document_loaders/naval-ravikant-book.pdf"; + const loader = new PDFLoader(bookPath); - // Chunk it + const docs = await loader.load(); - // Reduce the size of the metadata + if (docs.length === 0) { + console.log("No documents found."); + return; + } + + const splitter = new CharacterTextSplitter({ + separator: " ", + chunkSize: 250, + chunkOverlap: 10, + }); + + const splitDocs = await splitter.splitDocuments(docs); + + // Reduce the size of the metadata for each document -- lots of useless pdf information + const reducedDocs = splitDocs.map((doc) => { + const reducedMetadata = { ...doc.metadata }; + delete reducedMetadata.pdf; // Remove the 'pdf' field + return new Document({ + pageContent: doc.pageContent, + metadata: reducedMetadata, + }); + }); + + // docs.forEach((doc) => { + // console.log(doc); + // }); + + // console.log(`Uploading documents to Pinecone: ${docs}`); + + console.log(docs[100]); + console.log(splitDocs[100].metadata); + console.log(reducedDocs[100].metadata); /** STEP TWO: UPLOAD TO DATABASE */ - // upload documents to Pinecone - return res.status(200).json({ result: docs }); + const client = new PineconeClient(); + + await client.init({ + apiKey: process.env.PINECONE_API_KEY, + environment: process.env.PINECONE_ENVIRONMENT, + }); + + const pineconeIndex = client.Index(process.env.PINECONE_INDEX); + + await PineconeStore.fromDocuments(reducedDocs, new OpenAIEmbeddings(), { + pineconeIndex, + }); + + console.log("Successfully uploaded to DB"); + // Modify output as needed + return res.status(200).json({ + result: `Uploaded to Pinecone! Before splitting: ${docs.length}, After splitting: ${splitDocs.length}`, + }); } else { res.status(405).json({ message: "Method not allowed" }); } diff --git a/pages/api/resume-query-metadata.js b/pages/api/resume-query-metadata.js index 35eb61a..07e4f6f 100644 --- a/pages/api/resume-query-metadata.js +++ b/pages/api/resume-query-metadata.js @@ -12,12 +12,60 @@ import { OpenAI } from "langchain/llms/openai"; import { VectorDBQAChain } from "langchain/chains"; import { PromptTemplate } from "langchain/prompts"; +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ + export default async function handler(req, res) { try { // do stuff + const { prompt } = req.body; + + /** Load vector database */ + const client = new PineconeClient(); + await client.init({ + apiKey: process.env.PINECONE_API_KEY, + environment: process.env.PINECONE_ENVIRONMENT, + }); + + const pineconeIndex = client.Index(process.env.PINECONE_INDEX); + + const vectorStore = await PineconeStore.fromExistingIndex( + new OpenAIEmbeddings(), + { pineconeIndex } + ); + + // Create Vector DBQA CHain + const model = new OpenAI(); + const chain = VectorDBQAChain.fromLLM(model, vectorStore, { + k: 1, + returnSourceDocuments: true, + }); + + // Prompt Template + const promptTemplate = new PromptTemplate({ + template: `Assume you are a Human Resources Director. According to the resumes, answer this question: {question}`, + inputVariables: ["question"], + }); + + const formattedPrompt = await promptTemplate.format({ + question: prompt, + }); + + // console.log({ formattedPrompt }); + + const response = await chain.call({ + query: formattedPrompt, + }); + + console.log({ response }); return res.status(200).json({ + // String output: response.text, + // [Document, Document] sourceDocuments: response.sourceDocuments, }); } catch (err) { diff --git a/pages/api/resume-upload.js b/pages/api/resume-upload.js index da7369a..92b4fca 100644 --- a/pages/api/resume-upload.js +++ b/pages/api/resume-upload.js @@ -17,6 +17,12 @@ import { PineconeClient } from "@pinecone-database/pinecone"; import { loadSummarizationChain } from "langchain/chains"; import { OpenAI } from "langchain/llms/openai"; +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ + export default async function handler(req, res) { // Grab the prompt from the url (?prompt=[value]) // console.log(process.env.PINECONE_API_KEY); @@ -24,7 +30,96 @@ export default async function handler(req, res) { // console.log(process.env.PINECONE_INDEX); // Always use a try catch block to do asynchronous requests and catch any errors try { - // do stuff + // Load the directory + const loader = new DirectoryLoader( + "/Users/shawnesquivel/Desktop/openai-javascript-course/data/resumes", + { + ".pdf": (path) => new PDFLoader(path, "/pdf"), + } + ); + + const docs = await loader.load(); + // 3 + // console.log(`Loaded ${docs.length}`); + + // Split the documents with their metadata + const splitter = new CharacterTextSplitter({ + separator: " ", + chunkSize: 200, + chunkOverlap: 20, + }); + + // + const splitDocs = await splitter.splitDocuments(docs); + + // console.log(`Split Docs: ${splitDocs.length}`); + + // console.log(docs[0]); + // console.log(splitDocs[0]); + + // reduce the metadata and make it more searchable + const reducedDocs = splitDocs.map((doc) => { + // ["Users", "shawnesquivel", ... "resume_aubrey_graham.pdf"] + const fileName = doc.metadata.source.split("/").pop(); + // ["resume", "aubrey", "graham.pdf"] + const [_, firstName, lastName] = fileName.split("_"); + + return { + ...doc, + metadata: { + first_name: firstName, + last_name: lastName.slice(0, -4), + docType: "resume", + }, + }; + }); + + // console.log(reducedDocs[4]); + let summaries = []; + const model = new OpenAI({ temperature: 0 }); + const summarizeAllChain = loadSummarizationChain(model, { + type: "map_reduce", + }); + + // raw documents + const summarizeRes = await summarizeAllChain.call({ + input_documents: docs, + }); + summaries.push({ summary: summarizeRes.text }); + + /** Summarize each candidate */ + for (let doc of docs) { + const summarizeOneChain = loadSummarizationChain(model, { + type: "map_reduce", + }); + const summarizeOneRes = await summarizeOneChain.call({ + input_documents: [doc], + }); + + console.log({ summarizeOneRes }); + summaries.push({ summary: summarizeOneRes.text }); + } + + /** Upload the reducedDocs */ + const client = new PineconeClient(); + await client.init({ + apiKey: process.env.PINECONE_API_KEY, + environment: process.env.PINECONE_ENVIRONMENT, + }); + + const pineconeIndex = client.Index(process.env.PINECONE_INDEX); + + await PineconeStore.fromDocuments(reducedDocs, new OpenAIEmbeddings(), { + pineconeIndex, + }); + + console.log("Uploaded to Pinecone"); + + console.log({ summaries }); + // [{summary: 'gdajkljgadkl'}, {summary: 'gdjaklgkadl'}] + const summaryStr = JSON.stringify(summaries, null, 2); + + return res.status(200).json({ output: summaryStr }); } catch (err) { // If we have an error diff --git a/pages/api/solutions/chatcompletions-soln.js b/pages/api/solutions/chatcompletions-soln.js deleted file mode 100644 index 0d9e7c0..0000000 --- a/pages/api/solutions/chatcompletions-soln.js +++ /dev/null @@ -1,33 +0,0 @@ -import { ChatOpenAI } from "langchain/chat_models/openai"; -import { HumanChatMessage, SystemChatMessage } from "langchain/schema"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ - -const chat = new ChatOpenAI({ temperature: 0, modelName: "gpt-3.5-turbo" }); - -export default async function handler(req, res) { - if (req.method === "POST") { - // Grab the user prompt - const { input } = req.body; - - if (!input) { - throw new Error("No input"); - } - - // Enter your code here - const response = await chat.call([ - new HumanChatMessage(`How do I write a for loop in ${input}?`), - ]); - - console.log(response); - - // Modify output as needed - return res.status(200).json({ result: response }); - } else { - res.status(405).json({ message: "Method not allowed" }); - } -} diff --git a/pages/api/solutions/content-generator-soln.js b/pages/api/solutions/content-generator-soln.js deleted file mode 100644 index d5d7648..0000000 --- a/pages/api/solutions/content-generator-soln.js +++ /dev/null @@ -1,187 +0,0 @@ -// /pages/api/transcript_chat.js - -import { YoutubeTranscript } from "youtube-transcript"; -import extractVideoId from "../../utils/extractVideoId"; -import getVideoMetaData from "../../utils/getVideoMetaData"; -import { ChatOpenAI } from "langchain/chat_models/openai"; -import { LLMChain } from "langchain/chains"; -import ResearchAgent from "../../agents/ResearchAgent"; -import { - ChatPromptTemplate, - HumanMessagePromptTemplate, - SystemMessagePromptTemplate, -} from "langchain/prompts"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ - -// Global Variables -let chain; -let chatHistory = []; -let transcript = ""; -let metadataString = ""; -let research; - -// Initialize Chain with Data -const initChain = async (transcript, metadataString, research, topic) => { - try { - // For chat models, we provide a `ChatPromptTemplate` class that can be used to format chat prompts. - const llm = new ChatOpenAI({ - temperature: 0.7, - modelName: "gpt-3.5-turbo", - }); - - console.log(`Initializing Chat Prompt`); - - // For chat models, we provide a `ChatPromptTemplate` class that can be used to format chat prompts. - // This allows us to set the template that the bot sees every time - const chatPrompt = ChatPromptTemplate.fromPromptMessages([ - SystemMessagePromptTemplate.fromTemplate( - "You are a helpful social media assistant that provides research, new content, and advice to me. \n You are given the transcript of the video: {transcript} \n and video metadata: {metadata} as well as additional research: {research}" - ), - HumanMessagePromptTemplate.fromTemplate( - "{input}. Remember to use the video transcript and research as reference." - ), - ]); - - const question = `Write me a script for a new video that provides commentary on this video in a lighthearted, joking manner. It should compliment ${topic} with puns.`; - console.log(question); - - chain = new LLMChain({ - prompt: chatPrompt, - llm: llm, - // memory, - }); - - const response = await chain.call({ - transcript, - metadata: metadataString, - research, - input: question, - }); - - console.log({ response }); - - chatHistory.push({ - role: "assistant", - content: response.text, - }); - - return response; - } catch (error) { - console.error( - `An error occurred during the initialization of the Chat Prompt: ${error.message}` - ); - throw error; // rethrow the error to let the calling function know that an error occurred - } -}; - -export default async function handler(req, res) { - const { prompt, topic, firstMsg } = req.body; - console.log(`Prompt: ${prompt} Topic: ${topic}`); - - if ( - chain === undefined && - !prompt.includes("https://www.youtube.com/watch?v=") - ) { - return res.status(400).json({ - error: - "Chain not initialized. Please send a YouTube URL to initialize the chain.", - }); - } - - chatHistory.push({ - role: "user", - content: prompt, - }); - - // Just like in the previous section, if we have a firstMsg set to true, we need to initialize with chain with the context - if (firstMsg) { - console.log("Received URL"); - try { - const videoId = extractVideoId(prompt); - // API call for video transcript (same as last video, but we just grab the array and flatten it into a variable)[{text:" "},{ text: ""}] - const transcriptResponse = await YoutubeTranscript.fetchTranscript( - prompt - ); - transcriptResponse.forEach((line) => { - transcript += line.text; - }); - // Some error handling - if (!transcriptResponse) { - return res.status(400).json({ error: "Failed to get transcript" }); - } - - // API call for video metadata –– go to VideoMetaData and explain this - const metadata = await getVideoMetaData(videoId); - - // JSON object { [], [], [] } , null (no characters between), and use 2 spaces for indentation - metadataString = JSON.stringify(metadata, null, 2); - console.log({ metadataString }); - - // ResearchAgent - research = await ResearchAgent(topic); - - console.log({ research }); - - // Alright, finally we have all the context and we can initialize the chain! - const response = await initChain( - transcript, - metadataString, - research, - topic - ); - - // return res.status(200).json({ output: research }); - return res.status(200).json({ - output: response, - chatHistory, - transcript, - metadata, - research, - }); - } catch (err) { - console.error(err); - return res - .status(500) - .json({ error: "An error occurred while fetching transcript" }); - } - } else { - // Very similar to previous section, don't worry too much about this just copy and paste it from the previous section! - console.log("Received question"); - try { - const question = prompt; - - console.log("Asking:", question); - console.log("Using old chain:", chain); - // Everytime we call the chain we need to pass all the context back so that it can fill in the prompt template appropriately - const response = await chain.call({ - transcript, - metadata: metadataString, - research, - input: question, - }); - - // update chat history - chatHistory.push({ - role: "assistant", - content: response.text, - }); - // just make sure to modify this response as necessary. - return res.status(200).json({ - output: response, - metadata: metadataString, - transcript, - chatHistory, - }); - } catch (error) { - console.error(error); - res - .status(500) - .json({ error: "An error occurred during the conversation." }); - } - } -} diff --git a/pages/api/solutions/memory-soln.js b/pages/api/solutions/memory-soln.js deleted file mode 100644 index 8a806ed..0000000 --- a/pages/api/solutions/memory-soln.js +++ /dev/null @@ -1,37 +0,0 @@ -// Solution -import { OpenAI } from "langchain/llms/openai"; -import { BufferMemory } from "langchain/memory"; -import { ConversationChain } from "langchain/chains"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ - -let model; -let memory; -let chain; -export default async function handler(req, res) { - if (req.method === "POST") { - const { input, firstMsg } = req.body; - - if (!input) { - throw new Error("No input!"); - } - - if (firstMsg) { - console.log("initializing chain"); - model = new OpenAI({ modelName: "gpt-3.5-turbo" }); - memory = new BufferMemory(); - chain = new ConversationChain({ llm: model, memory: memory }); - } - - console.log({ input }); - const response = await chain.call({ input }); - console.log({ response }); - return res.status(200).json({ output: response }); - } else { - res.status(405).json({ message: "Only POST is allowed" }); - } -} diff --git a/pages/api/solutions/pdf-query-soln.js b/pages/api/solutions/pdf-query-soln.js deleted file mode 100644 index bb345c7..0000000 --- a/pages/api/solutions/pdf-query-soln.js +++ /dev/null @@ -1,58 +0,0 @@ -import { PineconeClient } from "@pinecone-database/pinecone"; -import { VectorDBQAChain } from "langchain/chains"; -import { OpenAIEmbeddings } from "langchain/embeddings/openai"; -import { OpenAI } from "langchain/llms/openai"; -import { PineconeStore } from "langchain/vectorstores/pinecone"; -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ - -// Example: https://js.langchain.com/docs/modules/indexes/document_loaders/examples/file_loaders/pdf -export default async function handler(req, res) { - try { - if (req.method !== "POST") { - throw new Error("Method not allowed"); - } - - console.log("Query PDF"); - - // Grab the user prompt - const { input } = req.body; - - if (!input) { - throw new Error("No input"); - } - - console.log("input received:", input); - - const client = new PineconeClient(); - await client.init({ - apiKey: process.env.PINECONE_API_KEY, - environment: process.env.PINECONE_ENVIRONMENT, - }); - const pineconeIndex = client.Index(process.env.PINECONE_INDEX); - - const vectorStore = await PineconeStore.fromExistingIndex( - new OpenAIEmbeddings(), - { pineconeIndex } - ); - - /* Part Two: Use as part of a chain (currently no metadata filters) */ - - const model = new OpenAI(); - const chain = VectorDBQAChain.fromLLM(model, vectorStore, { - k: 1, - returnSourceDocuments: true, - }); - const response = await chain.call({ query: input }); - - console.log(response); - - return res.status(200).json({ result: response }); - } catch (error) { - console.error(error); - res.status(500).json({ message: error.message }); - } -} diff --git a/pages/api/solutions/pdf-upload-soln.js b/pages/api/solutions/pdf-upload-soln.js deleted file mode 100644 index 2b808d4..0000000 --- a/pages/api/solutions/pdf-upload-soln.js +++ /dev/null @@ -1,123 +0,0 @@ -import { PDFLoader } from "langchain/document_loaders/fs/pdf"; -import { PineconeClient } from "@pinecone-database/pinecone"; -import { Document } from "langchain/document"; -import { OpenAIEmbeddings } from "langchain/embeddings/openai"; -import { PineconeStore } from "langchain/vectorstores/pinecone"; -import { CharacterTextSplitter } from "langchain/text_splitter"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ - -// Example: https://js.langchain.com/docs/modules/indexes/document_loaders/examples/file_loaders/pdf - -/** - * - * INSTRUCTIONS - * 1. Run with book - error { - name: 'PineconeError', - source: 'server', - message: 'PineconeClient: Error calling upsert: PineconeError: metadata size is 140052 bytes, which exceeds the limit of 40960 bytes per vector', - stack: '' - } - * 2. Explain why -- vector meta data sizes is too big. - Language Models are often limited by the amount of text that you can pass to them. Therefore, it is neccessary to split them up into smaller chunks. LangChain provides several utilities for doing so. - https://js.langchain.com/docs/modules/indexes/text_splitters/ - - Play with chunk sizes... too small and you can't understand. - Fine tune this to your liking. - More vectors = more $$ - - - 3. Pinecone size 1536 - https://platform.openai.com/docs/guides/embeddings/second-generation-models - - 4. Upsert metadata size -- add this after split Docs - - // Reduce the size of the metadata for each document - const reducedDocs = splitDocs.map(doc => { - const reducedMetadata = { ...doc.metadata }; - delete reducedMetadata.pdf; // Remove the 'pdf' field - return new Document({ - pageContent: doc.pageContent, - metadata: reducedMetadata, - }); -}); - - - - - - * */ - -export default async function handler(req, res) { - if (req.method === "GET") { - console.log("Uploading book"); - // Enter your code here - /** STEP ONE: LOAD DOCUMENT */ - const bookPath = - "/Users/shawnesquivel/GitHub/yt-script-generator/data/document_loaders/naval-ravikant-book.pdf"; - const loader = new PDFLoader(bookPath); - - const docs = await loader.load(); - - if (docs.length === 0) { - console.log("No documents found."); - return; - } - - const splitter = new CharacterTextSplitter({ - separator: " ", - chunkSize: 250, - chunkOverlap: 10, - }); - - const splitDocs = await splitter.splitDocuments(docs); - - // Reduce the size of the metadata for each document -- lots of useless pdf information - const reducedDocs = splitDocs.map((doc) => { - const reducedMetadata = { ...doc.metadata }; - delete reducedMetadata.pdf; // Remove the 'pdf' field - return new Document({ - pageContent: doc.pageContent, - metadata: reducedMetadata, - }); - }); - - // docs.forEach((doc) => { - // console.log(doc); - // }); - - // console.log(`Uploading documents to Pinecone: ${docs}`); - - console.log(docs[100]); - console.log(splitDocs[100].metadata); - console.log(reducedDocs[100].metadata); - - /** STEP TWO: UPLOAD TO DATABASE */ - - const client = new PineconeClient(); - - await client.init({ - apiKey: process.env.PINECONE_API_KEY, - environment: process.env.PINECONE_ENVIRONMENT, - }); - - const pineconeIndex = client.Index(process.env.PINECONE_INDEX); - - await PineconeStore.fromDocuments(reducedDocs, new OpenAIEmbeddings(), { - pineconeIndex, - }); - - console.log("Successfully uploaded to DB"); - // Modify output as needed - return res.status(200).json({ - result: `Uploaded to Pinecone! Before splitting: ${docs.length}, After splitting: ${splitDocs.length}`, - }); - } else { - res.status(405).json({ message: "Method not allowed" }); - } -} diff --git a/pages/api/solutions/resume-query-metadata-soln.js b/pages/api/solutions/resume-query-metadata-soln.js deleted file mode 100644 index 07e4f6f..0000000 --- a/pages/api/solutions/resume-query-metadata-soln.js +++ /dev/null @@ -1,75 +0,0 @@ -/** - * This endpoint is used to load the resumes into the chain, then upload them to the Pinecone database. - * Tutorial: https://js.langchain.com/docs/modules/indexes/document_loaders/examples/file_loaders/directory - * Summarization: https://js.langchain.com/docs/modules/chains/other_chains/summarization - * Dependencies: npm install pdf-parse - */ - -import { OpenAIEmbeddings } from "langchain/embeddings/openai"; -import { PineconeStore } from "langchain/vectorstores/pinecone"; -import { PineconeClient } from "@pinecone-database/pinecone"; -import { OpenAI } from "langchain/llms/openai"; -import { VectorDBQAChain } from "langchain/chains"; -import { PromptTemplate } from "langchain/prompts"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ - -export default async function handler(req, res) { - try { - // do stuff - const { prompt } = req.body; - - /** Load vector database */ - const client = new PineconeClient(); - await client.init({ - apiKey: process.env.PINECONE_API_KEY, - environment: process.env.PINECONE_ENVIRONMENT, - }); - - const pineconeIndex = client.Index(process.env.PINECONE_INDEX); - - const vectorStore = await PineconeStore.fromExistingIndex( - new OpenAIEmbeddings(), - { pineconeIndex } - ); - - // Create Vector DBQA CHain - const model = new OpenAI(); - const chain = VectorDBQAChain.fromLLM(model, vectorStore, { - k: 1, - returnSourceDocuments: true, - }); - - // Prompt Template - const promptTemplate = new PromptTemplate({ - template: `Assume you are a Human Resources Director. According to the resumes, answer this question: {question}`, - inputVariables: ["question"], - }); - - const formattedPrompt = await promptTemplate.format({ - question: prompt, - }); - - // console.log({ formattedPrompt }); - - const response = await chain.call({ - query: formattedPrompt, - }); - - console.log({ response }); - - return res.status(200).json({ - // String - output: response.text, - // [Document, Document] - sourceDocuments: response.sourceDocuments, - }); - } catch (err) { - console.error(err); - return res.status(500).json({ error: "Error" }); - } -} diff --git a/pages/api/solutions/resume-upload-soln.js b/pages/api/solutions/resume-upload-soln.js deleted file mode 100644 index 92b4fca..0000000 --- a/pages/api/solutions/resume-upload-soln.js +++ /dev/null @@ -1,129 +0,0 @@ -// /pages/api/resume_upload.js -// Import dependencies - -/** - * This endpoint is used to load the resumes into the chain, then upload them to the Pinecone database. - * Tutorial: https://js.langchain.com/docs/modules/indexes/document_loaders/examples/file_loaders/directory - * Summarization: https://js.langchain.com/docs/modules/chains/other_chains/summarization - * Dependencies: npm install pdf-parse - */ - -import { DirectoryLoader } from "langchain/document_loaders/fs/directory"; -import { PDFLoader } from "langchain/document_loaders/fs/pdf"; -import { CharacterTextSplitter } from "langchain/text_splitter"; -import { OpenAIEmbeddings } from "langchain/embeddings/openai"; -import { PineconeStore } from "langchain/vectorstores/pinecone"; -import { PineconeClient } from "@pinecone-database/pinecone"; -import { loadSummarizationChain } from "langchain/chains"; -import { OpenAI } from "langchain/llms/openai"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ - -export default async function handler(req, res) { - // Grab the prompt from the url (?prompt=[value]) - // console.log(process.env.PINECONE_API_KEY); - // console.log(process.env.PINECONE_ENVIRONMENT); - // console.log(process.env.PINECONE_INDEX); - // Always use a try catch block to do asynchronous requests and catch any errors - try { - // Load the directory - const loader = new DirectoryLoader( - "/Users/shawnesquivel/Desktop/openai-javascript-course/data/resumes", - { - ".pdf": (path) => new PDFLoader(path, "/pdf"), - } - ); - - const docs = await loader.load(); - // 3 - // console.log(`Loaded ${docs.length}`); - - // Split the documents with their metadata - const splitter = new CharacterTextSplitter({ - separator: " ", - chunkSize: 200, - chunkOverlap: 20, - }); - - // - const splitDocs = await splitter.splitDocuments(docs); - - // console.log(`Split Docs: ${splitDocs.length}`); - - // console.log(docs[0]); - // console.log(splitDocs[0]); - - // reduce the metadata and make it more searchable - const reducedDocs = splitDocs.map((doc) => { - // ["Users", "shawnesquivel", ... "resume_aubrey_graham.pdf"] - const fileName = doc.metadata.source.split("/").pop(); - // ["resume", "aubrey", "graham.pdf"] - const [_, firstName, lastName] = fileName.split("_"); - - return { - ...doc, - metadata: { - first_name: firstName, - last_name: lastName.slice(0, -4), - docType: "resume", - }, - }; - }); - - // console.log(reducedDocs[4]); - let summaries = []; - const model = new OpenAI({ temperature: 0 }); - const summarizeAllChain = loadSummarizationChain(model, { - type: "map_reduce", - }); - - // raw documents - const summarizeRes = await summarizeAllChain.call({ - input_documents: docs, - }); - summaries.push({ summary: summarizeRes.text }); - - /** Summarize each candidate */ - for (let doc of docs) { - const summarizeOneChain = loadSummarizationChain(model, { - type: "map_reduce", - }); - const summarizeOneRes = await summarizeOneChain.call({ - input_documents: [doc], - }); - - console.log({ summarizeOneRes }); - summaries.push({ summary: summarizeOneRes.text }); - } - - /** Upload the reducedDocs */ - const client = new PineconeClient(); - await client.init({ - apiKey: process.env.PINECONE_API_KEY, - environment: process.env.PINECONE_ENVIRONMENT, - }); - - const pineconeIndex = client.Index(process.env.PINECONE_INDEX); - - await PineconeStore.fromDocuments(reducedDocs, new OpenAIEmbeddings(), { - pineconeIndex, - }); - - console.log("Uploaded to Pinecone"); - - console.log({ summaries }); - // [{summary: 'gdajkljgadkl'}, {summary: 'gdjaklgkadl'}] - const summaryStr = JSON.stringify(summaries, null, 2); - - return res.status(200).json({ output: summaryStr }); - } catch (err) { - // If we have an error - - console.error(err); - return res.status(500).json({ error: err }); - } -} diff --git a/pages/api/solutions/streaming-soln.js b/pages/api/solutions/streaming-soln.js deleted file mode 100644 index 8d661a2..0000000 --- a/pages/api/solutions/streaming-soln.js +++ /dev/null @@ -1,46 +0,0 @@ -import { OpenAI } from "langchain/llms/openai"; -import SSE from "express-sse"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ - -const sse = new SSE(); - -export default function handler(req, res) { - if (req.method === "POST") { - const { input } = req.body; - - if (!input) { - throw new Error("No input"); - } - // Initialize model - const chat = new OpenAI({ - streaming: true, - callbacks: [ - { - handleLLMNewToken(token) { - sse.send(token, "newToken"); - }, - }, - ], - }); - - // create the prompt - const prompt = `Create me a short rap about my name and city. Make it funny and punny. Name: ${input}`; - - console.log({ prompt }); - // call frontend to backend - chat.call(prompt).then(() => { - sse.send(null, "end"); - }); - - return res.status(200).json({ result: "Streaming complete" }); - } else if (req.method === "GET") { - sse.init(req, res); - } else { - res.status(405).json({ message: "Method not allowed" }); - } -} diff --git a/pages/api/solutions/video-chat-soln.js b/pages/api/solutions/video-chat-soln.js deleted file mode 100644 index 4032d10..0000000 --- a/pages/api/solutions/video-chat-soln.js +++ /dev/null @@ -1,187 +0,0 @@ -// /pages/api/transcript.js -import { YoutubeTranscript } from "youtube-transcript"; -import { ChatOpenAI } from "langchain/chat_models/openai"; -import { ConversationalRetrievalQAChain } from "langchain/chains"; -import { HNSWLib } from "langchain/vectorstores/hnswlib"; -import { OpenAIEmbeddings } from "langchain/embeddings/openai"; -import { CharacterTextSplitter } from "langchain/text_splitter"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ - -// First, we'll initialize the chain and the chat history so that they can be preserved on multiple calls to the API -let chain; -// Remember, the chat history is where we store each human/chatbot message. -let chatHistory = []; - -// DO THIS SECOND -const initializeChain = async (initialPrompt, transcript) => { - try { - // Initialize model with GPT-3.5 - const model = new ChatOpenAI({ - temperature: 0.8, - modelName: "gpt-3.5-turbo", - }); - // Create a text splitter, we use a smaller chunk size and chunk overlap since we are working with small sentences - const splitter = new CharacterTextSplitter({ - separator: " ", - chunkSize: 7, - chunkOverlap: 3, - }); - - // Using the splitter, we create documents from a bigger document, in this case the YouTube Transcript - const docs = await splitter.createDocuments([transcript]); - - console.log(`Loading data ${docs[0]}`); - - // Upload chunks to database as documents - // We'll be using HNSWLib for this one. - // The nice thing about this one is that we don't need to create any accounts or get any API keys besides our OpenAI key to use this library - // So I find that it's nice for doing some quick prototyping. - // But the downside is that you don't get the nice dashboard like we had in Pinecone. - const vectorStore = await HNSWLib.fromDocuments( - [{ pageContent: transcript }], - new OpenAIEmbeddings() - ); - - // Just to show you, we'll also save the vector store as a file in case you want to retrieve it later. - // We'll copy our root directory and save it as a variable - const directory = "/Users/shawnesquivel/GitHub/yt-script-generator/"; - await vectorStore.save(directory); - // it will create some files for us, including a way for us to view the vector store documents which is helpful. - // then you can access it like this: - const loadedVectorStore = await HNSWLib.load( - directory, - new OpenAIEmbeddings() - ); - - // The ConversationalRetrievalQA chain builds on RetrievalQAChain to provide a chat history component. - - // To create one, you will need a retriever. In the below example, we will create one from a vectorstore, which can be created from embeddings. - - // Remember we can use the loadedVectorStore or the vectorStore, in case for example you want to scale this application up and use the same vector store to store multiple Youtube transcripts. - chain = ConversationalRetrievalQAChain.fromLLM( - model, - vectorStore.asRetriever(), - { verbose: true } // Add verbose option here - ); - - // It requires two inputs: a question and the chat history. It first combines the chat history and the question into a standalone question, then looks up relevant documents from the retriever, and then passes those documents and the question to a question answering chain to return a response. - const response = await chain.call({ - question: initialPrompt, - chat_history: chatHistory, - }); - - // Update history - chatHistory.push({ - role: "assistant", - content: response.text, - }); - - console.log({ chatHistory }); - return response; - } catch (error) { - console.error(error); - } -}; - -export default async function handler(req, res) { - if (req.method === "POST") { - // DO THIS FIRST - // First we'll destructure the prompt and firstMsg from the POST request body - const { prompt } = req.body; - const { firstMsg } = req.body; - - // Then if it's the first message, we want to initialize the chain, since it doesn't exist yet - if (firstMsg) { - console.log("Initializing chain"); - - try { - // So first of all, we want to give it our human message, which was to ask for a summary of the YouTube URL - const initialPrompt = `Give me a summary of the transcript: ${prompt}`; - - chatHistory.push({ - role: "user", - content: initialPrompt, - }); - - // Here, we'll use a generic YouTube Transcript API to get the transcript of a youtube video - // As you can see, the Transcript takes videoId/videoURL has the first argument to the function - const transcriptResponse = await YoutubeTranscript.fetchTranscript( - prompt - ); - - // and we'll just add some error handling in case the API fails - if (!transcriptResponse) { - return res.status(400).json({ error: "Failed to get transcript" }); - } - - // Now let's see what that transcriptResponse looks like - - console.log({ transcriptResponse }); - - // We can see that it's a big array of lines. Let's squish it down into one string first to make it easier to use. - - // We initialize the transcript string - let transcript = ""; - - // Then the forEach method calls each element in the array, e.g. line = element, and we can do something what that value - - // in this case, we'll add each line of text to the empty string variable to get a single string with the entire transcript - transcriptResponse.forEach((line) => { - transcript += line.text; - }); - - // Now, let's create a separate function called initialize chain - // We'll pass in the first prompt and the context, in this case the transcript - const response = await initializeChain(initialPrompt, transcript); - console.log("Chain:", chain); - console.log(response); - - // And then we'll jsut get the response back and the chatHistory - return res.status(200).json({ output: response, chatHistory }); - } catch (err) { - console.error(err); - return res - .status(500) - .json({ error: "An error occurred while fetching transcript" }); - } - - // DO THIS THIRD - } else { - // If it's not the first message, we can chat with the bot - console.log("Received question"); - try { - console.log("Asking:", prompt); - console.log("Chain:", chain); - - // First we'll add the user message - chatHistory.push({ - role: "user", - content: prompt, - }); - // Then we'll pass the entire chat history with all the previous messages back - const response = await chain.call({ - question: prompt, - chat_history: chatHistory, - }); - // And we'll add the response back as well - chatHistory.push({ - role: "assistant", - content: response.text, - }); - - return res.status(200).json({ output: response, chatHistory }); - } catch (error) { - // Generic error handling - console.error(error); - res - .status(500) - .json({ error: "An error occurred during the conversation." }); - } - } - } -} diff --git a/pages/api/streaming.js b/pages/api/streaming.js index 189016c..8d661a2 100644 --- a/pages/api/streaming.js +++ b/pages/api/streaming.js @@ -1,6 +1,12 @@ import { OpenAI } from "langchain/llms/openai"; import SSE from "express-sse"; +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ + const sse = new SSE(); export default function handler(req, res) { @@ -11,12 +17,27 @@ export default function handler(req, res) { throw new Error("No input"); } // Initialize model + const chat = new OpenAI({ + streaming: true, + callbacks: [ + { + handleLLMNewToken(token) { + sse.send(token, "newToken"); + }, + }, + ], + }); // create the prompt + const prompt = `Create me a short rap about my name and city. Make it funny and punny. Name: ${input}`; + console.log({ prompt }); // call frontend to backend + chat.call(prompt).then(() => { + sse.send(null, "end"); + }); - return res.status(200).json({ result: "OK" }); + return res.status(200).json({ result: "Streaming complete" }); } else if (req.method === "GET") { sse.init(req, res); } else { diff --git a/pages/api/video-chat.js b/pages/api/video-chat.js index 2b77589..4032d10 100644 --- a/pages/api/video-chat.js +++ b/pages/api/video-chat.js @@ -5,13 +5,82 @@ import { ConversationalRetrievalQAChain } from "langchain/chains"; import { HNSWLib } from "langchain/vectorstores/hnswlib"; import { OpenAIEmbeddings } from "langchain/embeddings/openai"; import { CharacterTextSplitter } from "langchain/text_splitter"; -import { OpenAI } from "langchain"; -// Global variables +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ + +// First, we'll initialize the chain and the chat history so that they can be preserved on multiple calls to the API +let chain; +// Remember, the chat history is where we store each human/chatbot message. +let chatHistory = []; // DO THIS SECOND const initializeChain = async (initialPrompt, transcript) => { try { + // Initialize model with GPT-3.5 + const model = new ChatOpenAI({ + temperature: 0.8, + modelName: "gpt-3.5-turbo", + }); + // Create a text splitter, we use a smaller chunk size and chunk overlap since we are working with small sentences + const splitter = new CharacterTextSplitter({ + separator: " ", + chunkSize: 7, + chunkOverlap: 3, + }); + + // Using the splitter, we create documents from a bigger document, in this case the YouTube Transcript + const docs = await splitter.createDocuments([transcript]); + + console.log(`Loading data ${docs[0]}`); + + // Upload chunks to database as documents + // We'll be using HNSWLib for this one. + // The nice thing about this one is that we don't need to create any accounts or get any API keys besides our OpenAI key to use this library + // So I find that it's nice for doing some quick prototyping. + // But the downside is that you don't get the nice dashboard like we had in Pinecone. + const vectorStore = await HNSWLib.fromDocuments( + [{ pageContent: transcript }], + new OpenAIEmbeddings() + ); + + // Just to show you, we'll also save the vector store as a file in case you want to retrieve it later. + // We'll copy our root directory and save it as a variable + const directory = "/Users/shawnesquivel/GitHub/yt-script-generator/"; + await vectorStore.save(directory); + // it will create some files for us, including a way for us to view the vector store documents which is helpful. + // then you can access it like this: + const loadedVectorStore = await HNSWLib.load( + directory, + new OpenAIEmbeddings() + ); + + // The ConversationalRetrievalQA chain builds on RetrievalQAChain to provide a chat history component. + + // To create one, you will need a retriever. In the below example, we will create one from a vectorstore, which can be created from embeddings. + + // Remember we can use the loadedVectorStore or the vectorStore, in case for example you want to scale this application up and use the same vector store to store multiple Youtube transcripts. + chain = ConversationalRetrievalQAChain.fromLLM( + model, + vectorStore.asRetriever(), + { verbose: true } // Add verbose option here + ); + + // It requires two inputs: a question and the chat history. It first combines the chat history and the question into a standalone question, then looks up relevant documents from the retriever, and then passes those documents and the question to a question answering chain to return a response. + const response = await chain.call({ + question: initialPrompt, + chat_history: chatHistory, + }); + + // Update history + chatHistory.push({ + role: "assistant", + content: response.text, + }); + console.log({ chatHistory }); return response; } catch (error) { @@ -22,10 +91,56 @@ const initializeChain = async (initialPrompt, transcript) => { export default async function handler(req, res) { if (req.method === "POST") { // DO THIS FIRST + // First we'll destructure the prompt and firstMsg from the POST request body + const { prompt } = req.body; + const { firstMsg } = req.body; // Then if it's the first message, we want to initialize the chain, since it doesn't exist yet - if (x) { + if (firstMsg) { + console.log("Initializing chain"); + try { + // So first of all, we want to give it our human message, which was to ask for a summary of the YouTube URL + const initialPrompt = `Give me a summary of the transcript: ${prompt}`; + + chatHistory.push({ + role: "user", + content: initialPrompt, + }); + + // Here, we'll use a generic YouTube Transcript API to get the transcript of a youtube video + // As you can see, the Transcript takes videoId/videoURL has the first argument to the function + const transcriptResponse = await YoutubeTranscript.fetchTranscript( + prompt + ); + + // and we'll just add some error handling in case the API fails + if (!transcriptResponse) { + return res.status(400).json({ error: "Failed to get transcript" }); + } + + // Now let's see what that transcriptResponse looks like + + console.log({ transcriptResponse }); + + // We can see that it's a big array of lines. Let's squish it down into one string first to make it easier to use. + + // We initialize the transcript string + let transcript = ""; + + // Then the forEach method calls each element in the array, e.g. line = element, and we can do something what that value + + // in this case, we'll add each line of text to the empty string variable to get a single string with the entire transcript + transcriptResponse.forEach((line) => { + transcript += line.text; + }); + + // Now, let's create a separate function called initialize chain + // We'll pass in the first prompt and the context, in this case the transcript + const response = await initializeChain(initialPrompt, transcript); + console.log("Chain:", chain); + console.log(response); + // And then we'll jsut get the response back and the chatHistory return res.status(200).json({ output: response, chatHistory }); } catch (err) { @@ -35,11 +150,30 @@ export default async function handler(req, res) { .json({ error: "An error occurred while fetching transcript" }); } - // do this third! + // DO THIS THIRD } else { // If it's not the first message, we can chat with the bot - + console.log("Received question"); try { + console.log("Asking:", prompt); + console.log("Chain:", chain); + + // First we'll add the user message + chatHistory.push({ + role: "user", + content: prompt, + }); + // Then we'll pass the entire chat history with all the previous messages back + const response = await chain.call({ + question: prompt, + chat_history: chatHistory, + }); + // And we'll add the response back as well + chatHistory.push({ + role: "assistant", + content: response.text, + }); + return res.status(200).json({ output: response, chatHistory }); } catch (error) { // Generic error handling diff --git a/tools/SerpAPI.js b/tools/SerpAPI.js index 3f67190..3fd6052 100644 --- a/tools/SerpAPI.js +++ b/tools/SerpAPI.js @@ -1,5 +1,22 @@ import { SerpAPI } from "langchain/tools"; -const SerpAPITool = () => {}; +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ + +const SerpAPITool = () => { + const serpAPI = new SerpAPI(process.env.SERPAPI_API_KEY, { + baseUrl: "http://localhost:3000/agents", + location: "Vancouver,British Columbia, Canada", + hl: "en", + gl: "us", + }); + serpAPI.returnDirect = true; + + return serpAPI; +}; export default SerpAPITool; +s; diff --git a/tools/WebBrowser.js b/tools/WebBrowser.js index 3a59c31..17a085f 100644 --- a/tools/WebBrowser.js +++ b/tools/WebBrowser.js @@ -2,8 +2,19 @@ import { WebBrowser } from "langchain/tools/webbrowser"; import { ChatOpenAI } from "langchain/chat_models/openai"; import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ const WebBrowserTool = () => { - // do stuff! + const model = new ChatOpenAI({ temperature: 0 }); + const embeddings = new OpenAIEmbeddings({}); + + const browser = new WebBrowser({ model, embeddings }); + browser.returnDirect = true; + + return browser; }; export default WebBrowserTool; diff --git a/tools/solutions/SerpAPI-soln.js b/tools/solutions/SerpAPI-soln.js deleted file mode 100644 index f89abe9..0000000 --- a/tools/solutions/SerpAPI-soln.js +++ /dev/null @@ -1,21 +0,0 @@ -import { SerpAPI } from "langchain/tools"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ - -const SerpAPITool = () => { - const serpAPI = new SerpAPI(process.env.SERPAPI_API_KEY, { - baseUrl: "http://localhost:3000/agents", - location: "Vancouver,British Columbia, Canada", - hl: "en", - gl: "us", - }); - serpAPI.returnDirect = true; - - return serpAPI; -}; - -export default SerpAPITool; diff --git a/tools/solutions/WebBrowser-soln.js b/tools/solutions/WebBrowser-soln.js deleted file mode 100644 index 17a085f..0000000 --- a/tools/solutions/WebBrowser-soln.js +++ /dev/null @@ -1,20 +0,0 @@ -import { WebBrowser } from "langchain/tools/webbrowser"; -import { ChatOpenAI } from "langchain/chat_models/openai"; -import { OpenAIEmbeddings } from "langchain/embeddings/openai"; - -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ -const WebBrowserTool = () => { - const model = new ChatOpenAI({ temperature: 0 }); - const embeddings = new OpenAIEmbeddings({}); - - const browser = new WebBrowser({ model, embeddings }); - browser.returnDirect = true; - - return browser; -}; - -export default WebBrowserTool; From 8aff86b6c839cb9558a7736f08fbbbb172afe605 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Sat, 10 Jun 2023 11:23:47 -0700 Subject: [PATCH 02/17] fill in solutions for utils --- utils/extractVideoId.js | 3 +- utils/getVideoMetaData.js | 31 +++++++++++++++++--- utils/solutions/extractVideoId-soln.js | 4 --- utils/solutions/getVideoMetadata-soln.js | 36 ------------------------ 4 files changed, 29 insertions(+), 45 deletions(-) delete mode 100644 utils/solutions/extractVideoId-soln.js delete mode 100644 utils/solutions/getVideoMetadata-soln.js diff --git a/utils/extractVideoId.js b/utils/extractVideoId.js index c59da5b..6dd2664 100644 --- a/utils/extractVideoId.js +++ b/utils/extractVideoId.js @@ -1,3 +1,4 @@ export default function extractVideoId(url) { - // do stuff + const urlParams = new URLSearchParams(new URL(url).search); + return urlParams.get("v"); } diff --git a/utils/getVideoMetaData.js b/utils/getVideoMetaData.js index dcc8f33..f666ada 100644 --- a/utils/getVideoMetaData.js +++ b/utils/getVideoMetaData.js @@ -1,13 +1,36 @@ import axios from "axios"; export default async function getVideoMetaData(videoId) { - // enable api key and setup next.config.js + // First, we need to make sure we have our GOOGLE_API_KEY set up + // https://console.cloud.google.com/apis/ + // Look up the YouTube Data API V3 + // Enable API key + // Copy into .env as GOOGLE_API_KEY + // Configure Next.Config.JS const url = `https://www.googleapis.com/youtube/v3/videos?id=${videoId}&key=${process.env.GOOGLE_API_KEY}&part=snippet,contentDetails,statistics,status`; try { - // { data: {items: [metadata]}} + // HTTP request { data: {items: [metadata ]}} + const response = await axios.get(url); + const data = response.data; + const metadata = data.items[0]; + + console.log("GetMetadata", { metadata }); + // Clean up the response - } catch (err) { - console.error(`Failed to get metadata: ${err}`); + const videoTitle = metadata.snippet.title; + const videoDescription = metadata.snippet.description; + const shortenedDescription = videoDescription.split(".")[0]; + + const videoId = metadata.id; + // Create a small metadata object to return + const shortMetadata = { + videoTitle, + videoDescription: shortenedDescription, + videoId, + }; + return shortMetadata; // returns the first item, which should be the video if the id is valid + } catch (error) { + console.error(`Failed to fetch video metadata: ${error}`); } } diff --git a/utils/solutions/extractVideoId-soln.js b/utils/solutions/extractVideoId-soln.js deleted file mode 100644 index 6dd2664..0000000 --- a/utils/solutions/extractVideoId-soln.js +++ /dev/null @@ -1,4 +0,0 @@ -export default function extractVideoId(url) { - const urlParams = new URLSearchParams(new URL(url).search); - return urlParams.get("v"); -} diff --git a/utils/solutions/getVideoMetadata-soln.js b/utils/solutions/getVideoMetadata-soln.js deleted file mode 100644 index f666ada..0000000 --- a/utils/solutions/getVideoMetadata-soln.js +++ /dev/null @@ -1,36 +0,0 @@ -import axios from "axios"; - -export default async function getVideoMetaData(videoId) { - // First, we need to make sure we have our GOOGLE_API_KEY set up - // https://console.cloud.google.com/apis/ - // Look up the YouTube Data API V3 - // Enable API key - // Copy into .env as GOOGLE_API_KEY - // Configure Next.Config.JS - const url = `https://www.googleapis.com/youtube/v3/videos?id=${videoId}&key=${process.env.GOOGLE_API_KEY}&part=snippet,contentDetails,statistics,status`; - - try { - // HTTP request { data: {items: [metadata ]}} - const response = await axios.get(url); - const data = response.data; - const metadata = data.items[0]; - - console.log("GetMetadata", { metadata }); - - // Clean up the response - const videoTitle = metadata.snippet.title; - const videoDescription = metadata.snippet.description; - const shortenedDescription = videoDescription.split(".")[0]; - - const videoId = metadata.id; - // Create a small metadata object to return - const shortMetadata = { - videoTitle, - videoDescription: shortenedDescription, - videoId, - }; - return shortMetadata; // returns the first item, which should be the video if the id is valid - } catch (error) { - console.error(`Failed to fetch video metadata: ${error}`); - } -} From 75f8540241bddeb1f5e2af11fd305033f134180f Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Sat, 10 Jun 2023 11:26:11 -0700 Subject: [PATCH 03/17] word --- tools/WebBrowser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/WebBrowser.js b/tools/WebBrowser.js index 17a085f..0097e9e 100644 --- a/tools/WebBrowser.js +++ b/tools/WebBrowser.js @@ -4,7 +4,7 @@ import { OpenAIEmbeddings } from "langchain/embeddings/openai"; /** * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this! * */ const WebBrowserTool = () => { From 05d2593dd3d5d565dda27e24d717ac1802af57c6 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Sat, 10 Jun 2023 12:09:49 -0700 Subject: [PATCH 04/17] add streaming prototype --- app/streaming-v2/page.jsx | 46 ++++++++++++++++++++++++++++++++ pages/api/streamDemo.js | 56 +++++++++++++++++++++++++++++++++++++++ pages/api/streaming.js | 14 +++++++--- 3 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 app/streaming-v2/page.jsx create mode 100644 pages/api/streamDemo.js diff --git a/app/streaming-v2/page.jsx b/app/streaming-v2/page.jsx new file mode 100644 index 0000000..9ee7ad2 --- /dev/null +++ b/app/streaming-v2/page.jsx @@ -0,0 +1,46 @@ +/* + Copyright (c) 2023 J. Krajewski and released under the MIT license. +*/ +"use client"; +import { useState } from "react"; + +function Chatbot() { + const [chatMessages, setChatMessages] = useState([]); + + const handleButtonClick = async () => { + setChatMessages([]); + + const response = await fetch("/api/streamDemo"); + const stream = response.body; + const reader = stream.getReader(); + + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + const decodedValue = new TextDecoder().decode(value); + setChatMessages((prevMessages) => [...prevMessages, decodedValue]); + } + } catch (error) { + console.error(error); + } finally { + reader.releaseLock(); + } + }; + + return ( + <> + <button + className="text-lg font-bold text-slate-500" + onClick={handleButtonClick} + > + Start Stream + </button> + <ul>{chatMessages && chatMessages}</ul> + </> + ); +} + +export default Chatbot; diff --git a/pages/api/streamDemo.js b/pages/api/streamDemo.js new file mode 100644 index 0000000..2670124 --- /dev/null +++ b/pages/api/streamDemo.js @@ -0,0 +1,56 @@ +/* + Copyright (c) 2023 J. Krajewski and released under the MIT license. +*/ + +import { ChatOpenAI } from "langchain/chat_models"; +import { HumanChatMessage, SystemChatMessage } from "langchain/schema"; +import { CallbackManager } from "langchain/callbacks"; +const OPENAI_API_KEY = process.env.OPENAI_API_KEY; + +export default async function handler(req, res) { + try { + res.writeHead(200, { + "Content-Type": "application/octet-stream", + "Transfer-Encoding": "chunked", + }); + + if (!OPENAI_API_KEY) { + throw new Error("OPENAI_API_KEY is not defined."); + } + + let s = ""; + const chatStreaming = new ChatOpenAI({ + streaming: true, + callbackManager: CallbackManager.fromHandlers({ + async handleLLMNewToken(token) { + // console.clear(); + // s += token; + // console.log(s); + handleNewToken(token); + }, + }), + }); + + // const responseD = await chatStreaming.call([ + // new HumanChatMessage("Write me a song about sparkling water."), + // ]); + + function handleNewToken(token) { + res.write(`${token}`); + } + + await chatStreaming.call([ + new SystemChatMessage( + "You are a poet like tennyson, kipling, frost. You're famous and enjoyed by many." + ), + new HumanChatMessage( + "Write me a poem that rhymes about traveling from Mississippi to London for the first time." + ), + ]); + + res.end(); + } catch (error) { + console.error(error); + res.status(500).send("Internal Server Error"); + } +} diff --git a/pages/api/streaming.js b/pages/api/streaming.js index 8d661a2..1d976ff 100644 --- a/pages/api/streaming.js +++ b/pages/api/streaming.js @@ -33,9 +33,17 @@ export default function handler(req, res) { console.log({ prompt }); // call frontend to backend - chat.call(prompt).then(() => { - sse.send(null, "end"); - }); + chat + .call(prompt) + .then(() => { + sse.send(null, "end"); + }) + .catch((err) => { + console.error(err); + res + .status(500) + .json({ error: "An error occurred during the chat call" }); + }); return res.status(200).json({ result: "Streaming complete" }); } else if (req.method === "GET") { From 896af0ab38cf20f9a487f659a8736a0da98a6d9e Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Sat, 10 Jun 2023 13:41:31 -0700 Subject: [PATCH 05/17] remove file system dependencies --- pages/api/video-chat.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/pages/api/video-chat.js b/pages/api/video-chat.js index 4032d10..581bffd 100644 --- a/pages/api/video-chat.js +++ b/pages/api/video-chat.js @@ -47,17 +47,6 @@ const initializeChain = async (initialPrompt, transcript) => { new OpenAIEmbeddings() ); - // Just to show you, we'll also save the vector store as a file in case you want to retrieve it later. - // We'll copy our root directory and save it as a variable - const directory = "/Users/shawnesquivel/GitHub/yt-script-generator/"; - await vectorStore.save(directory); - // it will create some files for us, including a way for us to view the vector store documents which is helpful. - // then you can access it like this: - const loadedVectorStore = await HNSWLib.load( - directory, - new OpenAIEmbeddings() - ); - // The ConversationalRetrievalQA chain builds on RetrievalQAChain to provide a chat history component. // To create one, you will need a retriever. In the below example, we will create one from a vectorstore, which can be created from embeddings. From ca39f86afb37ed965b4ffa8236ccd105a84d3b46 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Sat, 10 Jun 2023 14:00:14 -0700 Subject: [PATCH 06/17] add error handling --- app/content-generator/page.jsx | 6 ++++ pages/api/content-generator.js | 51 ++++++++++++++++------------------ 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/app/content-generator/page.jsx b/app/content-generator/page.jsx index fdea40f..62b12d4 100644 --- a/app/content-generator/page.jsx +++ b/app/content-generator/page.jsx @@ -51,6 +51,12 @@ const ContentGenerator = () => { }, body: JSON.stringify({ prompt: prompt, topic: topic, firstMsg }), }); + if (!response.ok) { + console.error(`Error: ${response.status}`); // log the HTTP status code + const errorMessage = await response.text(); // get the error message from the response + console.error(`Message: ${errorMessage}`); // log the error message + throw new Error(errorMessage); + } const searchRes = await response.json(); diff --git a/pages/api/content-generator.js b/pages/api/content-generator.js index d5d7648..eabaad4 100644 --- a/pages/api/content-generator.js +++ b/pages/api/content-generator.js @@ -17,6 +17,19 @@ import { * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. * */ +// /pages/api/transcript_chat.js + +import { YoutubeTranscript } from "youtube-transcript"; +import extractVideoId from "../../utils/extractVideoId"; +import getVideoMetaData from "../../utils/getVideoMetaData"; +import { ChatOpenAI } from "langchain/chat_models/openai"; +import { LLMChain } from "langchain/chains"; +import ResearchAgent from "../../agents/ResearchAgent"; +import { + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +} from "langchain/prompts"; // Global Variables let chain; @@ -25,10 +38,8 @@ let transcript = ""; let metadataString = ""; let research; -// Initialize Chain with Data const initChain = async (transcript, metadataString, research, topic) => { try { - // For chat models, we provide a `ChatPromptTemplate` class that can be used to format chat prompts. const llm = new ChatOpenAI({ temperature: 0.7, modelName: "gpt-3.5-turbo", @@ -36,8 +47,6 @@ const initChain = async (transcript, metadataString, research, topic) => { console.log(`Initializing Chat Prompt`); - // For chat models, we provide a `ChatPromptTemplate` class that can be used to format chat prompts. - // This allows us to set the template that the bot sees every time const chatPrompt = ChatPromptTemplate.fromPromptMessages([ SystemMessagePromptTemplate.fromTemplate( "You are a helpful social media assistant that provides research, new content, and advice to me. \n You are given the transcript of the video: {transcript} \n and video metadata: {metadata} as well as additional research: {research}" @@ -53,7 +62,6 @@ const initChain = async (transcript, metadataString, research, topic) => { chain = new LLMChain({ prompt: chatPrompt, llm: llm, - // memory, }); const response = await chain.call({ @@ -75,7 +83,7 @@ const initChain = async (transcript, metadataString, research, topic) => { console.error( `An error occurred during the initialization of the Chat Prompt: ${error.message}` ); - throw error; // rethrow the error to let the calling function know that an error occurred + throw new Error(`initChain error: ${error.message}`); } }; @@ -98,36 +106,29 @@ export default async function handler(req, res) { content: prompt, }); - // Just like in the previous section, if we have a firstMsg set to true, we need to initialize with chain with the context if (firstMsg) { console.log("Received URL"); try { const videoId = extractVideoId(prompt); - // API call for video transcript (same as last video, but we just grab the array and flatten it into a variable)[{text:" "},{ text: ""}] + const transcriptResponse = await YoutubeTranscript.fetchTranscript( prompt ); + if (!transcriptResponse) { + throw new Error("Failed to get transcript"); + } + transcriptResponse.forEach((line) => { transcript += line.text; }); - // Some error handling - if (!transcriptResponse) { - return res.status(400).json({ error: "Failed to get transcript" }); - } - // API call for video metadata –– go to VideoMetaData and explain this const metadata = await getVideoMetaData(videoId); - - // JSON object { [], [], [] } , null (no characters between), and use 2 spaces for indentation metadataString = JSON.stringify(metadata, null, 2); console.log({ metadataString }); - // ResearchAgent research = await ResearchAgent(topic); - console.log({ research }); - // Alright, finally we have all the context and we can initialize the chain! const response = await initChain( transcript, metadataString, @@ -135,7 +136,6 @@ export default async function handler(req, res) { topic ); - // return res.status(200).json({ output: research }); return res.status(200).json({ output: response, chatHistory, @@ -147,17 +147,15 @@ export default async function handler(req, res) { console.error(err); return res .status(500) - .json({ error: "An error occurred while fetching transcript" }); + .json({ error: `Error fetching transcript: ${err.message}` }); } } else { - // Very similar to previous section, don't worry too much about this just copy and paste it from the previous section! console.log("Received question"); try { const question = prompt; - console.log("Asking:", question); console.log("Using old chain:", chain); - // Everytime we call the chain we need to pass all the context back so that it can fill in the prompt template appropriately + const response = await chain.call({ transcript, metadata: metadataString, @@ -165,12 +163,11 @@ export default async function handler(req, res) { input: question, }); - // update chat history chatHistory.push({ role: "assistant", content: response.text, }); - // just make sure to modify this response as necessary. + return res.status(200).json({ output: response, metadata: metadataString, @@ -179,9 +176,9 @@ export default async function handler(req, res) { }); } catch (error) { console.error(error); - res + return res .status(500) - .json({ error: "An error occurred during the conversation." }); + .json({ error: `Error during conversation: ${error.message}` }); } } } From e2411bef75f3a5d674e71ec310d0b35f89c1b151 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Sat, 10 Jun 2023 14:03:34 -0700 Subject: [PATCH 07/17] remove extra imports --- pages/api/content-generator.js | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pages/api/content-generator.js b/pages/api/content-generator.js index eabaad4..deed731 100644 --- a/pages/api/content-generator.js +++ b/pages/api/content-generator.js @@ -19,18 +19,6 @@ import { */ // /pages/api/transcript_chat.js -import { YoutubeTranscript } from "youtube-transcript"; -import extractVideoId from "../../utils/extractVideoId"; -import getVideoMetaData from "../../utils/getVideoMetaData"; -import { ChatOpenAI } from "langchain/chat_models/openai"; -import { LLMChain } from "langchain/chains"; -import ResearchAgent from "../../agents/ResearchAgent"; -import { - ChatPromptTemplate, - HumanMessagePromptTemplate, - SystemMessagePromptTemplate, -} from "langchain/prompts"; - // Global Variables let chain; let chatHistory = []; From ea7476a9b30501681ada869b7ddb4ef404261066 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Sat, 10 Jun 2023 14:13:23 -0700 Subject: [PATCH 08/17] attempt to fix reference error for content-generator --- next.config.js | 7 ++++++- package.json | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/next.config.js b/next.config.js index 6319174..eebf4ea 100644 --- a/next.config.js +++ b/next.config.js @@ -10,7 +10,12 @@ const nextConfig = { return config; }, - // Add env { API_KEY: process.env.API_KEY} + env: { OPENAI_API_KEY: process.env.OPENAI_API_KEY }, + // To fix Reference Error from content-generator: https://github.com/vercel/next.js/issues/40399 + experimental: { + browsersListForSwc: true, + legacyBrowsers: false, + }, }; module.exports = nextConfig; diff --git a/package.json b/package.json index ec4a30c..7f66ce6 100644 --- a/package.json +++ b/package.json @@ -36,5 +36,11 @@ "@types/react": "18.2.6", "express-sse": "^0.5.3", "typescript": "5.0.4" - } + }, + "browserslist": [ + "chrome 105", + "edge 105", + "ios_saf 15.6", + "firefox 104" + ] } From 993348cec6c1752898a246c81e4e373682c85db1 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Sat, 10 Jun 2023 14:20:41 -0700 Subject: [PATCH 09/17] remove THE S --- tools/SerpAPI.js | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/SerpAPI.js b/tools/SerpAPI.js index 3fd6052..f89abe9 100644 --- a/tools/SerpAPI.js +++ b/tools/SerpAPI.js @@ -19,4 +19,3 @@ const SerpAPITool = () => { }; export default SerpAPITool; -s; From 38e89c367ae16cde72843af796f21f6e3864d7c1 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Sat, 10 Jun 2023 14:34:33 -0700 Subject: [PATCH 10/17] remove streaming, testing pages --- app/components/HamburgerMenu.jsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/components/HamburgerMenu.jsx b/app/components/HamburgerMenu.jsx index a2d2928..a319bc6 100644 --- a/app/components/HamburgerMenu.jsx +++ b/app/components/HamburgerMenu.jsx @@ -108,7 +108,7 @@ const HamburgerMenu = () => { </span> </a> </li> - <li className="flex flex-col gap-4"> + {/* <li className="flex flex-col gap-4"> <a href="streaming"> <span> <div className=" rounded-xl overflow-hidden h-40 w-32 drop-shadow"> @@ -129,7 +129,7 @@ const HamburgerMenu = () => { </p> </span> </a> - </li> + </li> */} <li className="flex flex-col gap-4"> <a href="video-chat"> <span> @@ -196,7 +196,7 @@ const HamburgerMenu = () => { </span> </a> </li> - <li className="flex flex-col gap-4"> + {/* <li className="flex flex-col gap-4"> <a href="chatcompletions"> <span> <div className=" rounded-xl overflow-hidden h-40 w-32 drop-shadow"> @@ -217,7 +217,7 @@ const HamburgerMenu = () => { </p> </span> </a> - </li> + </li> */} </ul> </nav> </div> From 83545cb3878b322c7d73772355ca2c63a53e3b01 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Sat, 10 Jun 2023 14:46:16 -0700 Subject: [PATCH 11/17] allow error to persist on init chain --- pages/api/content-generator.js | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pages/api/content-generator.js b/pages/api/content-generator.js index deed731..402c00c 100644 --- a/pages/api/content-generator.js +++ b/pages/api/content-generator.js @@ -51,13 +51,21 @@ const initChain = async (transcript, metadataString, research, topic) => { prompt: chatPrompt, llm: llm, }); + let response; - const response = await chain.call({ - transcript, - metadata: metadataString, - research, - input: question, - }); + try { + response = await chain.call({ + transcript, + metadata: metadataString, + research, + input: question, + }); + } catch (error) { + console.error( + `An error occurred during the call to chain: ${error.message}` + ); + response = { text: "" }; // Provide an empty response or a default value + } console.log({ response }); From 7368090a09f8daa7e3c0a94a2cf6311ef473f9ee Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Mon, 12 Jun 2023 10:30:55 -0700 Subject: [PATCH 12/17] add supabase solutions --- next.config.js | 12 +- package-lock.json | 221 +++++++++++++++++++- package.json | 1 + pages/api/resume-query-metadata-supabase.js | 77 +++++++ pages/api/resume-upload-supabase-soln.js | 149 +++++++++++++ 5 files changed, 457 insertions(+), 3 deletions(-) create mode 100644 pages/api/resume-query-metadata-supabase.js create mode 100644 pages/api/resume-upload-supabase-soln.js diff --git a/next.config.js b/next.config.js index eebf4ea..4647c83 100644 --- a/next.config.js +++ b/next.config.js @@ -10,7 +10,17 @@ const nextConfig = { return config; }, - env: { OPENAI_API_KEY: process.env.OPENAI_API_KEY }, + env: { + OPENAI_API_KEY: process.env.OPENAI_API_KEY, + SERPAPI_API_KEY: process.env.SERPAPI_API_KEY, + GOOGLE_API_KEY: process.env.GOOGLE_API_KEY, + PINECONE_API_KEY: process.env.PINECONE_API_KEY, + PINECONE_ENV: process.env.PINECONE_ENV, + PINECONE_INDEX: process.env.PINECONE_INDEX, + GOOGLE_API_KEY: process.env.GOOGLE_API_KEY, + SUPABASE_URL: process.env.SUPABASE_URL, + SUPABASE_PRIVATE_KEY: process.env.SUPABASE_PRIVATE_KEY, + }, // To fix Reference Error from content-generator: https://github.com/vercel/next.js/issues/40399 experimental: { browsersListForSwc: true, diff --git a/package-lock.json b/package-lock.json index 01111bb..f9a1b60 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "0.1.0", "dependencies": { "@pinecone-database/pinecone": "^0.1.5", + "@supabase/supabase-js": "^2.24.0", "autoprefixer": "10.4.14", "axios": "^1.4.0", "cheerio": "^1.0.0-rc.12", @@ -301,6 +302,61 @@ "node": ">=14.0.0" } }, + "node_modules/@supabase/functions-js": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@supabase/functions-js/-/functions-js-2.1.1.tgz", + "integrity": "sha512-bIR1Puae6W+1/MzPfYBWOG/SCWGo4B5CB7c0ZZksvliNEAzhxNBJ0UFKYINcGdGtxG8ZC+1xr3utWpNZNwnoRw==", + "dependencies": { + "cross-fetch": "^3.1.5" + } + }, + "node_modules/@supabase/gotrue-js": { + "version": "2.29.0", + "resolved": "https://registry.npmjs.org/@supabase/gotrue-js/-/gotrue-js-2.29.0.tgz", + "integrity": "sha512-QJepUxSXpgcyMhDtEusRyGtCcYSqy4wRDf3BQGqLUDaU/sRRclO07NCHW8nBqGW6KZZ6oNLfKX2AQz621dmIPw==", + "dependencies": { + "cross-fetch": "^3.1.5" + } + }, + "node_modules/@supabase/postgrest-js": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-1.7.0.tgz", + "integrity": "sha512-wLADHZ5jm7LljF4GigK0H2vc1wGupBY2hGYfb4fVo0UuyMftmA6tOYy+ZpMH/vPq01CUFwXGwvIke6kyqh/QDg==", + "dependencies": { + "cross-fetch": "^3.1.5" + } + }, + "node_modules/@supabase/realtime-js": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/@supabase/realtime-js/-/realtime-js-2.7.2.tgz", + "integrity": "sha512-Fi6xAl5PUkqnjl3wo4rdcQIbMG3+yTRX1aUZe/yfvTG84RMvmCXJ1yN6MmafVLeZpU1xkaz5Vx4L0tnHcLiy6w==", + "dependencies": { + "@types/phoenix": "^1.5.4", + "@types/websocket": "^1.0.3", + "websocket": "^1.0.34" + } + }, + "node_modules/@supabase/storage-js": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.5.1.tgz", + "integrity": "sha512-nkR0fQA9ScAtIKA3vNoPEqbZv1k5B5HVRYEvRWdlP6mUpFphM9TwPL2jZ/ztNGMTG5xT6SrHr+H7Ykz8qzbhjw==", + "dependencies": { + "cross-fetch": "^3.1.5" + } + }, + "node_modules/@supabase/supabase-js": { + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.24.0.tgz", + "integrity": "sha512-zrAm+hp6DBICqZ7xVPk+KofmlfjJWQzXuf2sHAyPz8XVjpha84z2OVWcow2aI10YkMOrPwhRtBBQYJOnh/fx2w==", + "dependencies": { + "@supabase/functions-js": "^2.1.0", + "@supabase/gotrue-js": "^2.26.0", + "@supabase/postgrest-js": "^1.7.0", + "@supabase/realtime-js": "^2.7.2", + "@supabase/storage-js": "^2.5.1", + "cross-fetch": "^3.1.5" + } + }, "node_modules/@swc/helpers": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.1.tgz", @@ -321,8 +377,12 @@ "node_modules/@types/node": { "version": "20.1.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.0.tgz", - "integrity": "sha512-O+z53uwx64xY7D6roOi4+jApDGFg0qn6WHcxe5QeqjMaTezBO/mxdfFXIVAVVyNWKx84OmPB3L8kbVYOTeN34A==", - "dev": true + "integrity": "sha512-O+z53uwx64xY7D6roOi4+jApDGFg0qn6WHcxe5QeqjMaTezBO/mxdfFXIVAVVyNWKx84OmPB3L8kbVYOTeN34A==" + }, + "node_modules/@types/phoenix": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@types/phoenix/-/phoenix-1.6.0.tgz", + "integrity": "sha512-qwfpsHmFuhAS/dVd4uBIraMxRd56vwBUYQGZ6GpXnFuM2XMRFJbIyruFKKlW2daQliuYZwe0qfn/UjFCDKic5g==" }, "node_modules/@types/prop-types": { "version": "15.7.5", @@ -352,6 +412,14 @@ "integrity": "sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ==", "dev": true }, + "node_modules/@types/websocket": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/websocket/-/websocket-1.0.5.tgz", + "integrity": "sha512-NbsqiNX9CnEfC1Z0Vf4mE1SgAJ07JnRYcNex7AJ9zAVzmiGHmjKFEk7O4TJIsgv2B1sLEb6owKFZrACwdYngsQ==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/amdefine": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", @@ -565,6 +633,18 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/bufferutil": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.7.tgz", + "integrity": "sha512-kukuqc39WOHtdxtw4UScxF/WVnMFVSQVKhtx3AjZJzhd0RGZZldcrfSEbVsWWe6KNH253574cq5F+wpv0G9pJw==", + "hasInstallScript": true, + "dependencies": { + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=6.14.2" + } + }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -778,6 +858,15 @@ "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==", "dev": true }, + "node_modules/d": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", + "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", + "dependencies": { + "es5-ext": "^0.10.50", + "type": "^1.0.1" + } + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -887,6 +976,39 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, + "node_modules/es5-ext": { + "version": "0.10.62", + "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.62.tgz", + "integrity": "sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==", + "hasInstallScript": true, + "dependencies": { + "es6-iterator": "^2.0.3", + "es6-symbol": "^3.1.3", + "next-tick": "^1.1.0" + }, + "engines": { + "node": ">=0.10" + } + }, + "node_modules/es6-iterator": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", + "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==", + "dependencies": { + "d": "1", + "es5-ext": "^0.10.35", + "es6-symbol": "^3.1.1" + } + }, + "node_modules/es6-symbol": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", + "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", + "dependencies": { + "d": "^1.0.1", + "ext": "^1.1.2" + } + }, "node_modules/escalade": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", @@ -964,6 +1086,19 @@ "integrity": "sha512-DJF0nofFGq0IXJLGq95hfrryP3ZprVAVpyZUnmAk6QhHnm7zCzsHBNFP0i4FKFo2XjOf+JiYUKjT7jQhIeljpg==", "dev": true }, + "node_modules/ext": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", + "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==", + "dependencies": { + "type": "^2.7.2" + } + }, + "node_modules/ext/node_modules/type": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz", + "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==" + }, "node_modules/fast-glob": { "version": "3.2.12", "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", @@ -1226,6 +1361,11 @@ "node": ">=0.12.0" } }, + "node_modules/is-typedarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", + "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" + }, "node_modules/jiti": { "version": "1.18.2", "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.18.2.tgz", @@ -1601,6 +1741,11 @@ } } }, + "node_modules/next-tick": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", + "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==" + }, "node_modules/next/node_modules/postcss": { "version": "8.4.14", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.14.tgz", @@ -1653,6 +1798,16 @@ } } }, + "node_modules/node-gyp-build": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.0.tgz", + "integrity": "sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ==", + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-releases": { "version": "2.0.10", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.10.tgz", @@ -2376,6 +2531,19 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" }, + "node_modules/type": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", + "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" + }, + "node_modules/typedarray-to-buffer": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", + "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", + "dependencies": { + "is-typedarray": "^1.0.0" + } + }, "node_modules/typescript": { "version": "5.0.4", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.0.4.tgz", @@ -2429,6 +2597,18 @@ "browserslist": ">= 4.21.0" } }, + "node_modules/utf-8-validate": { + "version": "5.0.10", + "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz", + "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==", + "hasInstallScript": true, + "dependencies": { + "node-gyp-build": "^4.3.0" + }, + "engines": { + "node": ">=6.14.2" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -2455,6 +2635,35 @@ "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, + "node_modules/websocket": { + "version": "1.0.34", + "resolved": "https://registry.npmjs.org/websocket/-/websocket-1.0.34.tgz", + "integrity": "sha512-PRDso2sGwF6kM75QykIesBijKSVceR6jL2G8NGYyq2XrItNC2P5/qL5XeR056GhA+Ly7JMFvJb9I312mJfmqnQ==", + "dependencies": { + "bufferutil": "^4.0.1", + "debug": "^2.2.0", + "es5-ext": "^0.10.50", + "typedarray-to-buffer": "^3.1.5", + "utf-8-validate": "^5.0.2", + "yaeti": "^0.0.6" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/websocket/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/websocket/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, "node_modules/whatwg-url": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", @@ -2469,6 +2678,14 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, + "node_modules/yaeti": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/yaeti/-/yaeti-0.0.6.tgz", + "integrity": "sha512-MvQa//+KcZCUkBTIC9blM+CU9J2GzuTytsOUwf2lidtvkx/6gnEp1QvJv34t9vdjhFmha/mUiNDbN0D0mJWdug==", + "engines": { + "node": ">=0.10.32" + } + }, "node_modules/yaml": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.2.tgz", diff --git a/package.json b/package.json index 7f66ce6..4ec22c7 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ }, "dependencies": { "@pinecone-database/pinecone": "^0.1.5", + "@supabase/supabase-js": "^2.24.0", "autoprefixer": "10.4.14", "axios": "^1.4.0", "cheerio": "^1.0.0-rc.12", diff --git a/pages/api/resume-query-metadata-supabase.js b/pages/api/resume-query-metadata-supabase.js new file mode 100644 index 0000000..e1aff04 --- /dev/null +++ b/pages/api/resume-query-metadata-supabase.js @@ -0,0 +1,77 @@ +/** + * This endpoint is used to load the resumes into the chain, then upload them to the Pinecone database. + * Tutorial: https://js.langchain.com/docs/modules/indexes/document_loaders/examples/file_loaders/directory + * Summarization: https://js.langchain.com/docs/modules/chains/other_chains/summarization + * Dependencies: npm install pdf-parse + */ + +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { PineconeStore } from "langchain/vectorstores/pinecone"; +import { PineconeClient } from "@pinecone-database/pinecone"; +import { OpenAI } from "langchain/llms/openai"; +import { VectorDBQAChain } from "langchain/chains"; +import { PromptTemplate } from "langchain/prompts"; +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { createClient } from "@supabase/supabase-js"; +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ + +export default async function handler(req, res) { + try { + // do stuff + const { prompt } = req.body; + + /** Load vector database */ + /** CONNECT TO SUPABASE */ + const privateKey = process.env.SUPABASE_PRIVATE_KEY; + if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + + const url = process.env.SUPABASE_URL; + if (!url) throw new Error(`Expected env var SUPABASE_URL`); + + const client = createClient(url, privateKey); + + const vectorStore = await SupabaseVectorStore.fromExistingIndex( + new OpenAIEmbeddings(), + { client, tableName: "documents", queryName: "match_documents" } + ); + + // Create Vector DBQA CHain + const model = new OpenAI(); + const chain = VectorDBQAChain.fromLLM(model, vectorStore, { + k: 1, + returnSourceDocuments: true, + }); + + // Prompt Template + const promptTemplate = new PromptTemplate({ + template: `Assume you are a Human Resources Director. According to the resumes, answer this question: {question}`, + inputVariables: ["question"], + }); + + const formattedPrompt = await promptTemplate.format({ + question: prompt, + }); + + // console.log({ formattedPrompt }); + + const response = await chain.call({ + query: formattedPrompt, + }); + + console.log({ response }); + + return res.status(200).json({ + // String + output: response.text, + // [Document, Document] + sourceDocuments: response.sourceDocuments, + }); + } catch (err) { + console.error(err); + return res.status(500).json({ error: "Error" }); + } +} diff --git a/pages/api/resume-upload-supabase-soln.js b/pages/api/resume-upload-supabase-soln.js new file mode 100644 index 0000000..6d585e9 --- /dev/null +++ b/pages/api/resume-upload-supabase-soln.js @@ -0,0 +1,149 @@ +// /pages/api/resume_upload.js +// Import dependencies + +/** + * This endpoint is used to load the resumes into the chain, then upload them to the Pinecone database. + * Tutorial: https://js.langchain.com/docs/modules/indexes/document_loaders/examples/file_loaders/directory + * Summarization: https://js.langchain.com/docs/modules/chains/other_chains/summarization + * Dependencies: npm install pdf-parse + */ + +import { DirectoryLoader } from "langchain/document_loaders/fs/directory"; +import { PDFLoader } from "langchain/document_loaders/fs/pdf"; +import { CharacterTextSplitter } from "langchain/text_splitter"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { loadSummarizationChain } from "langchain/chains"; +import { OpenAI } from "langchain/llms/openai"; +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { createClient } from "@supabase/supabase-js"; + +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ + +export default async function handler(req, res) { + // Grab the prompt from the url (?prompt=[value]) + // console.log(process.env.PINECONE_API_KEY); + // console.log(process.env.PINECONE_ENVIRONMENT); + // console.log(process.env.PINECONE_INDEX); + // Always use a try catch block to do asynchronous requests and catch any errors + try { + // Load the directory + const loader = new DirectoryLoader( + "/Users/shawnesquivel/Desktop/openai-javascript-course/data/resumes", + { + ".pdf": (path) => new PDFLoader(path, "/pdf"), + } + ); + + const docs = await loader.load(); + // 3 + // console.log(`Loaded ${docs.length}`); + + // Split the documents with their metadata + const splitter = new CharacterTextSplitter({ + separator: " ", + chunkSize: 200, + chunkOverlap: 20, + }); + + // + const splitDocs = await splitter.splitDocuments(docs); + + // console.log(`Split Docs: ${splitDocs.length}`); + + // console.log(docs[0]); + // console.log(splitDocs[0]); + + // reduce the metadata and make it more searchable + const reducedDocs = splitDocs.map((doc) => { + // ["Users", "shawnesquivel", ... "resume_aubrey_graham.pdf"] + const fileName = doc.metadata.source.split("/").pop(); + // ["resume", "aubrey", "graham.pdf"] + const [_, firstName, lastName] = fileName.split("_"); + + return { + ...doc, + metadata: { + first_name: firstName, + last_name: lastName.slice(0, -4), + docType: "resume", + }, + }; + }); + + // console.log(reducedDocs[4]); + let summaries = []; + const model = new OpenAI({ temperature: 0 }); + const summarizeAllChain = loadSummarizationChain(model, { + type: "map_reduce", + }); + + // raw documents + const summarizeRes = await summarizeAllChain.call({ + input_documents: docs, + }); + summaries.push({ summary: summarizeRes.text }); + + /** Summarize each candidate */ + for (let doc of docs) { + const summarizeOneChain = loadSummarizationChain(model, { + type: "map_reduce", + }); + const summarizeOneRes = await summarizeOneChain.call({ + input_documents: [doc], + }); + + console.log({ summarizeOneRes }); + summaries.push({ summary: summarizeOneRes.text }); + } + + /** Upload the reducedDocs */ + + // STEP ONE: REMOVE THE PINECONE CODE + // const client = new PineconeClient(); + // await client.init({ + // apiKey: process.env.PINECONE_API_KEY, + // environment: process.env.PINECONE_ENVIRONMENT, + // }); + + // const pineconeIndex = client.Index(process.env.PINECONE_INDEX); + + // await PineconeStore.fromDocuments(reducedDocs, new OpenAIEmbeddings(), { + // pineconeIndex, + // }); + + /** STEP TWO: REPLACE WITH SUPABASE CLIENT */ + /** CONNECT TO SUPABASE */ + const privateKey = process.env.SUPABASE_PRIVATE_KEY; + if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + + const url = process.env.SUPABASE_URL; + if (!url) throw new Error(`Expected env var SUPABASE_URL`); + + const client = createClient(url, privateKey); + + /** UPLOAD TO SUPABASE */ + await SupabaseVectorStore.fromDocuments( + reducedDocs, + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + console.log({ summaries }); + // [{summary: 'gdajkljgadkl'}, {summary: 'gdjaklgkadl'}] + const summaryStr = JSON.stringify(summaries, null, 2); + + return res.status(200).json({ output: summaryStr }); + } catch (err) { + // If we have an error + + console.error(err); + return res.status(500).json({ error: err }); + } +} From 0a127e3a1016702c21e18414dabe075b7904a523 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Sat, 17 Jun 2023 11:40:08 -0700 Subject: [PATCH 13/17] add supabase solutions --- pages/api/pdf-query.js | 26 +++- pages/api/pdf-upload.js | 20 +++ pages/api/resume-upload-supabase-soln.js | 16 +-- pages/api/resume-upload-supabase.js | 154 +++++++++++++++++++++++ playground/supabase.mjs | 127 +++++++++++++++++++ 5 files changed, 322 insertions(+), 21 deletions(-) create mode 100644 pages/api/resume-upload-supabase.js create mode 100644 playground/supabase.mjs diff --git a/pages/api/pdf-query.js b/pages/api/pdf-query.js index bb345c7..8b50d57 100644 --- a/pages/api/pdf-query.js +++ b/pages/api/pdf-query.js @@ -27,18 +27,32 @@ export default async function handler(req, res) { console.log("input received:", input); - const client = new PineconeClient(); - await client.init({ - apiKey: process.env.PINECONE_API_KEY, - environment: process.env.PINECONE_ENVIRONMENT, - }); - const pineconeIndex = client.Index(process.env.PINECONE_INDEX); + // const client = new PineconeClient(); + // await client.init({ + // apiKey: process.env.PINECONE_API_KEY, + // environment: process.env.PINECONE_ENVIRONMENT, + // }); + // const pineconeIndex = client.Index(process.env.PINECONE_INDEX); const vectorStore = await PineconeStore.fromExistingIndex( new OpenAIEmbeddings(), { pineconeIndex } ); + const privateKey = process.env.SUPABASE_PRIVATE_KEY; + if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + + const url = process.env.SUPABASE_URL; + if (!url) throw new Error(`Expected env var SUPABASE_URL`); + + const client = createClient(url, privateKey); + + // Alternative: use Supabase instead of Pinecone if you're on waitlist (see this video for explanation: https://www.udemy.com/course/langchain-develop-ai-web-apps-with-javascript-and-langchain/learn/lecture/38362160) + // const vectorStore = await SupabaseVectorStore.fromExistingIndex( + // new OpenAIEmbeddings(), + // { client, tableName: "documents", queryName: "match_documents" } + // ); + /* Part Two: Use as part of a chain (currently no metadata filters) */ const model = new OpenAI(); diff --git a/pages/api/pdf-upload.js b/pages/api/pdf-upload.js index 2b808d4..fbd5532 100644 --- a/pages/api/pdf-upload.js +++ b/pages/api/pdf-upload.js @@ -112,6 +112,26 @@ export default async function handler(req, res) { pineconeIndex, }); + // Alternative: use Supabase instead of Pinecone if you're on waitlist (see this video for explanation: https://www.udemy.com/course/langchain-develop-ai-web-apps-with-javascript-and-langchain/learn/lecture/38362160) + // const privateKey = process.env.SUPABASE_PRIVATE_KEY; + // if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + + // const url = process.env.SUPABASE_URL; + // if (!url) throw new Error(`Expected env var SUPABASE_URL`); + + // const client = createClient(url, privateKey); + + /** UPLOAD TO SUPABASE */ + await SupabaseVectorStore.fromDocuments( + reducedDocs, + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + console.log("Successfully uploaded to DB"); // Modify output as needed return res.status(200).json({ diff --git a/pages/api/resume-upload-supabase-soln.js b/pages/api/resume-upload-supabase-soln.js index 6d585e9..c9a4067 100644 --- a/pages/api/resume-upload-supabase-soln.js +++ b/pages/api/resume-upload-supabase-soln.js @@ -101,21 +101,7 @@ export default async function handler(req, res) { } /** Upload the reducedDocs */ - - // STEP ONE: REMOVE THE PINECONE CODE - // const client = new PineconeClient(); - // await client.init({ - // apiKey: process.env.PINECONE_API_KEY, - // environment: process.env.PINECONE_ENVIRONMENT, - // }); - - // const pineconeIndex = client.Index(process.env.PINECONE_INDEX); - - // await PineconeStore.fromDocuments(reducedDocs, new OpenAIEmbeddings(), { - // pineconeIndex, - // }); - - /** STEP TWO: REPLACE WITH SUPABASE CLIENT */ + /** STEP TWO: REPLACE WITH SUPABASE CLIENT INSTEAD OF PINECONE */ /** CONNECT TO SUPABASE */ const privateKey = process.env.SUPABASE_PRIVATE_KEY; if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); diff --git a/pages/api/resume-upload-supabase.js b/pages/api/resume-upload-supabase.js new file mode 100644 index 0000000..6e28973 --- /dev/null +++ b/pages/api/resume-upload-supabase.js @@ -0,0 +1,154 @@ +// /pages/api/resume_upload.js +// Import dependencies + +/** + * This endpoint is used to load the resumes into the chain, then upload them to the Pinecone database. + * Tutorial: https://js.langchain.com/docs/modules/indexes/document_loaders/examples/file_loaders/directory + * Summarization: https://js.langchain.com/docs/modules/chains/other_chains/summarization + * Dependencies: npm install pdf-parse + */ + +import { DirectoryLoader } from "langchain/document_loaders/fs/directory"; +import { PDFLoader } from "langchain/document_loaders/fs/pdf"; +import { CharacterTextSplitter } from "langchain/text_splitter"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { PineconeStore } from "langchain/vectorstores/pinecone"; +import { PineconeClient } from "@pinecone-database/pinecone"; +import { loadSummarizationChain } from "langchain/chains"; +import { OpenAI } from "langchain/llms/openai"; +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { createClient } from "@supabase/supabase-js"; + +/** + * + * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. + * + */ + +export default async function handler(req, res) { + // Grab the prompt from the url (?prompt=[value]) + // console.log(process.env.PINECONE_API_KEY); + // console.log(process.env.PINECONE_ENVIRONMENT); + // console.log(process.env.PINECONE_INDEX); + // Always use a try catch block to do asynchronous requests and catch any errors + try { + // Load the directory + const loader = new DirectoryLoader( + "/Users/shawnesquivel/Desktop/openai-javascript-course/data/resumes", + { + ".pdf": (path) => new PDFLoader(path, "/pdf"), + } + ); + + const docs = await loader.load(); + // 3 + // console.log(`Loaded ${docs.length}`); + + // Split the documents with their metadata + const splitter = new CharacterTextSplitter({ + separator: " ", + chunkSize: 200, + chunkOverlap: 20, + }); + + // + const splitDocs = await splitter.splitDocuments(docs); + + // console.log(`Split Docs: ${splitDocs.length}`); + + // console.log(docs[0]); + // console.log(splitDocs[0]); + + // reduce the metadata and make it more searchable + const reducedDocs = splitDocs.map((doc) => { + // ["Users", "shawnesquivel", ... "resume_aubrey_graham.pdf"] + const fileName = doc.metadata.source.split("/").pop(); + // ["resume", "aubrey", "graham.pdf"] + const [_, firstName, lastName] = fileName.split("_"); + + return { + ...doc, + metadata: { + first_name: firstName, + last_name: lastName.slice(0, -4), + docType: "resume", + }, + }; + }); + + // console.log(reducedDocs[4]); + let summaries = []; + const model = new OpenAI({ temperature: 0 }); + const summarizeAllChain = loadSummarizationChain(model, { + type: "map_reduce", + }); + + // raw documents + const summarizeRes = await summarizeAllChain.call({ + input_documents: docs, + }); + summaries.push({ summary: summarizeRes.text }); + + /** Summarize each candidate */ + for (let doc of docs) { + const summarizeOneChain = loadSummarizationChain(model, { + type: "map_reduce", + }); + const summarizeOneRes = await summarizeOneChain.call({ + input_documents: [doc], + }); + + console.log({ summarizeOneRes }); + summaries.push({ summary: summarizeOneRes.text }); + } + + /** Upload the reducedDocs */ + // PINECONE VERSION + // const client = new PineconeClient(); + // await client.init({ + // apiKey: process.env.PINECONE_API_KEY, + // environment: process.env.PINECONE_ENVIRONMENT, + // }); + + // const pineconeIndex = client.Index(process.env.PINECONE_INDEX); + + // await PineconeStore.fromDocuments(reducedDocs, new OpenAIEmbeddings(), { + // pineconeIndex, + // }); + + // console.log("Uploaded to Pinecone"); + + // SUPABASE VERSION + /** STEP TWO: REPLACE WITH SUPABASE CLIENT */ + /** CONNECT TO SUPABASE */ + const privateKey = process.env.SUPABASE_PRIVATE_KEY; + if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + + const url = process.env.SUPABASE_URL; + if (!url) throw new Error(`Expected env var SUPABASE_URL`); + + const client = createClient(url, privateKey); + + /** UPLOAD TO SUPABASE */ + await SupabaseVectorStore.fromDocuments( + reducedDocs, + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } + ); + + console.log({ summaries }); + // [{summary: 'gdajkljgadkl'}, {summary: 'gdjaklgkadl'}] + const summaryStr = JSON.stringify(summaries, null, 2); + + return res.status(200).json({ output: summaryStr }); + } catch (err) { + // If we have an error + + console.error(err); + return res.status(500).json({ error: err }); + } +} diff --git a/playground/supabase.mjs b/playground/supabase.mjs new file mode 100644 index 0000000..0fbdb85 --- /dev/null +++ b/playground/supabase.mjs @@ -0,0 +1,127 @@ +// Supabase x LangChainJS https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase + +// How Supabase Works https://supabase.com/blog/openai-embeddings-postgres-vector + +// NOTE: This is an alternative to Pinecone (waitlist due to high demand) +// 1: Create account on supabase +// https://app.supabase.com/ + +// 2. Create table named `documents` +import { OpenAI } from "langchain/llms/openai"; +import { SupabaseVectorStore } from "langchain/vectorstores/supabase"; +import { OpenAIEmbeddings } from "langchain/embeddings/openai"; +import { createClient } from "@supabase/supabase-js"; +import { VectorDBQAChain } from "langchain/chains"; +import { PromptTemplate } from "langchain/prompts"; + +// First, follow set-up instructions at +// https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase + +// https://supabase.com/blog/openai-embeddings-postgres-vector#using-postgresql +console.log(process.env.SUPABASE_PRIVATE_KEY); +console.log(process.env.SUPABASE_URL); +console.log(process.env.OPENAI_API_KEY); + +/** STEP 1: CONNECT TO SUPABASE */ +const privateKey = process.env.SUPABASE_PRIVATE_KEY; +if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); + +const url = process.env.SUPABASE_URL; +if (!url) throw new Error(`Expected env var SUPABASE_URL`); + +const client = createClient(url, privateKey); + +/** STEP 2: UPLOAD STUFF */ +// The SupabaseVectorStore.fromTexts method takes four arguments: the texts to store, +// the associated metadata, the embedding function, and options for interacting with the database. +const vectorStor1e = await SupabaseVectorStore.fromTexts( + [ + "Frontend Developer Python ", + "Backend Developer Java", + "AI/ML Researcher LangChain", + "AI/ML Researcher Meta", + ], // These are the texts you want to store in the vector store. Each text will be converted into a high-dimensional vector using an embedding function. + [ + { first_name: "Joanna", last_name: "Smith" }, + { first_name: "Kaito", last_name: "Esquivel" }, + { first_name: "Aubrey", last_name: "Graham" }, + { first_name: "Andres", last_name: "Tatum" }, + ], // This is the metadata associated with each text. It's an array of objects, with each object corresponding to a text. The metadata can be anything you want and is not used in the embedding process, but it can be useful for identifying the texts later. + new OpenAIEmbeddings(), // This is the function that will convert each text into a high-dimensional vector. In this case, you're using the OpenAIEmbeddings function, which uses OpenAI's GPT-3 model to create the embeddings. + { + client, // This is the Supabase client, which was created earlier using the createClient function. This client is used to interact with your Supabase database. + tableName: "documents", // This is the name of the table in your Supabase database where the vectors will be stored. + queryName: "match_documents", // This is the name of the function in your database that will be used to find the most similar vectors to a given vector. You must have previously defined this function in your database. + } +); + +const vectorStore = await SupabaseVectorStore.fromTexts( + [ + "Frontend Developer Python", + "Backend Developer Java", + "AI/ML Researcher LangChain", + "AI/ML Researcher Meta", + ], + [ + { first_name: "Joanna", last_name: "Smith" }, + { first_name: "Kaito", last_name: "Esquivel" }, + { first_name: "Aubrey", last_name: "Graham" }, + { first_name: "Andres", last_name: "Tatum" }, + ], + new OpenAIEmbeddings(), + { + client, + tableName: "documents", + queryName: "match_documents", + } +); + +/** Step 3: Use a ConversationalRetrievalQAChain to interact */ +const model = new OpenAI({}); + +const chain = VectorDBQAChain.fromLLM(model, vectorStore, { + k: 1, + returnSourceDocuments: true, +}); + +const promptTemplate = new PromptTemplate({ + template: `Assume you are a Human Resources Director. According to the resumes, answer this question: {question}`, + inputVariables: ["question"], +}); + +const prompt = "Who has experience with Python?"; + +const formattedPrompt = await promptTemplate.format({ + question: prompt, +}); + +const response = await chain.call({ + query: formattedPrompt, +}); + +console.log(prompt); +console.log(response); +console.log(response.sourceDocuments[0].metadata); + +/** STEP 3: SIMILARITY SEARCH */ +// similaritySearch(query: string, k?: number, filter?: this["FilterType"] | undefined): Promise<Document[]>; +// const similarityRes = await vectorStore.similaritySearch("Python", 1); +// console.log({ similarityRes }); +// console.log(similarityRes.metadata); + +/** STEP 4: METADATA FILTER */ +// https://js.langchain.com/docs/modules/indexes/vector_stores/integrations/supabase#metadata-filtering +// Same thing, just add a third parameter for the metadata filter! +// We have two AI/ML researchers, but we want to get Andres +// const metadataRes = await vectorStore.similaritySearch("Hello world", 1, { +// first_name: "Andres", +// }); + +// console.log({ metadataRes }); +// console.log(metadataRes.metadata); + +// "No storage option exists to persist the session, which may result in unexpected behavior when using auth." +// - this warning is displayed because the Supabase client is trying to persist the user session but cannot find a storage option. +// This isn't an issue in your case as we are not dealing with user authentication. You can safely ignore this warning. +// You can modify this later if you are going to have authentication in your app – https://supabase.com/docs/guides/auth +// We will not cover it in this course, but Supabase is a GREAT option for it! From ffcb74daa9f25a5771515a593520fb61239d4d61 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Thu, 22 Jun 2023 21:05:00 -0700 Subject: [PATCH 14/17] next crash course solutions --- app/components/Emoji.jsx | 17 +++++++++++ app/nextjs/page.jsx | 64 ++++++++++++++++++++++++++++++++++++++++ pages/api/next.js | 14 +++++++++ 3 files changed, 95 insertions(+) create mode 100644 app/components/Emoji.jsx create mode 100644 app/nextjs/page.jsx create mode 100644 pages/api/next.js diff --git a/app/components/Emoji.jsx b/app/components/Emoji.jsx new file mode 100644 index 0000000..93452a9 --- /dev/null +++ b/app/components/Emoji.jsx @@ -0,0 +1,17 @@ +import React from "react"; + +const Emoji = ({ color }) => { + console.log(color); + return ( + <div className={`bg-${color}-500 text-center`}> + {/* <p>The color is {color}</p> */} + {/* Ternary Operator – If Else Statement ? : */} + {/* {CONDITION ? IF_VALUE : ELSE_VALUE } */} + + {color === "red" ? <p>The color is red</p> : <p>The color is not red</p>} + <p>🤖</p> + </div> + ); +}; + +export default Emoji; diff --git a/app/nextjs/page.jsx b/app/nextjs/page.jsx new file mode 100644 index 0000000..bd4869d --- /dev/null +++ b/app/nextjs/page.jsx @@ -0,0 +1,64 @@ +"use client"; +import React, { useState } from "react"; +import Emoji from "../components/Emoji"; + +// React Functional Component +const NextJSTutorial = () => { + // Logic, functions, data goes + const firstName = "Bryan"; + + const [lastName, setLastName] = useState(""); + + const handleSubmit = async () => { + console.log("woo hoo"); + + const response = await fetch("api/next", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ key: "Some Message", lastName }), + }); + + // JSON = JavaScript Object Notation {} + console.log(response); + + const responseJSON = await response.json(); + + console.log(responseJSON); + }; + + // Each component returns some JSX => allows us to write HTML in React BETTER! + return ( + <div> + <p>This is where the page appears</p> + <p>TailWind CSS is awesome</p> + <p className="red-font">{firstName}</p> + + {/* CTRL / or CMD / */} + + {/* STATE */} + <div className="flex flex-col space-y-4"> + <div> + <p>My last name is: {lastName}</p> + <input + type="text" + className="outline w-32 rounded-md" + onChange={(e) => { + setLastName(e.target.value); + }} + /> + + <button onClick={handleSubmit}>Submit</button> + </div> + </div> + + {/* EMOJI */} + <Emoji color="green" /> + <Emoji color="green" /> + <Emoji color="green" /> + </div> + ); +}; + +export default NextJSTutorial; diff --git a/pages/api/next.js b/pages/api/next.js new file mode 100644 index 0000000..ce3a80f --- /dev/null +++ b/pages/api/next.js @@ -0,0 +1,14 @@ +export default function handler(req, res) { + // We want to read key: "Some Message", lastName + console.log("im in the api route"); + const { lastName, key } = req.body; + + console.log(lastName); + console.log(key); + + if (lastName === "Zuckerberg") { + console.log("Meta is awesome"); + } + + res.status(200).json({ result: `Your last name ${lastName} is awesome` }); +} From 508079314820b0bf9aa92129ca9f335b66b97b16 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Tue, 27 Jun 2023 10:58:55 -0700 Subject: [PATCH 15/17] add back streaming into hamburger menu --- app/components/HamburgerMenu.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/components/HamburgerMenu.jsx b/app/components/HamburgerMenu.jsx index a319bc6..cbaa41f 100644 --- a/app/components/HamburgerMenu.jsx +++ b/app/components/HamburgerMenu.jsx @@ -108,7 +108,7 @@ const HamburgerMenu = () => { </span> </a> </li> - {/* <li className="flex flex-col gap-4"> + <li className="flex flex-col gap-4"> <a href="streaming"> <span> <div className=" rounded-xl overflow-hidden h-40 w-32 drop-shadow"> @@ -129,7 +129,7 @@ const HamburgerMenu = () => { </p> </span> </a> - </li> */} + </li> <li className="flex flex-col gap-4"> <a href="video-chat"> <span> From dd411c6f65851dd03358d2c167dca4d44a529474 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Tue, 18 Jun 2024 16:33:59 -0700 Subject: [PATCH 16/17] MAJOR UPDATE: langchain updates for 2024 --- app/pdf/page.jsx | 6 - package-lock.json | 2620 +++++++++++++++++++++++++++++++++++---------- package.json | 14 +- 3 files changed, 2044 insertions(+), 596 deletions(-) diff --git a/app/pdf/page.jsx b/app/pdf/page.jsx index 0236206..8c41cbe 100644 --- a/app/pdf/page.jsx +++ b/app/pdf/page.jsx @@ -113,12 +113,6 @@ const PDFLoader = () => { get started!" /> <ButtonContainer> - {/* <Button - handleSubmit={()=>{handleSubmit('pdfupload-book')}} - endpoint="pdfuploadtest" - buttonText="Upload Test Data ☁️" - className="Button" - /> */} <Button handleSubmit={handleSubmit} endpoint="pdf-upload" diff --git a/package-lock.json b/package-lock.json index f9a1b60..0f4b777 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,11 @@ "name": "yt-script-generator", "version": "0.1.0", "dependencies": { - "@pinecone-database/pinecone": "^0.1.5", + "@langchain/community": "^0.2.11", + "@langchain/openai": "^0.1.3", + "@langchain/pinecone": "^0.0.7", + "@langchain/textsplitters": "^0.0.3", + "@pinecone-database/pinecone": "^2.2.2", "@supabase/supabase-js": "^2.24.0", "autoprefixer": "10.4.14", "axios": "^1.4.0", @@ -16,8 +20,7 @@ "cors": "^2.8.5", "debug": "^4.3.4", "dotenv": "^16.0.3", - "hnswlib-node": "^1.4.2", - "langchain": "^0.0.75", + "langchain": "^0.2.5", "next": "13.4.1", "openai": "^3.2.1", "pdf-parse": "^1.1.1", @@ -49,75 +52,936 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.4.3.tgz", - "integrity": "sha512-SZrlXvjUUYT9rPmSzlTtmVk1OjVNpkCzILRluhiYwNcxXfQyvPJDi0CI6PyymygcgtqEF5EVqhKmC/PtPsNEIw==", + "node_modules/@fastify/busboy": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-2.1.1.tgz", + "integrity": "sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==", + "engines": { + "node": ">=14" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.5.tgz", + "integrity": "sha512-IzL8ZoEDIBRWEzlCcRhOaCupYyN5gdIK+Q6fbFdPDg6HqX6jpkItn7DFIpW9LQzXG6Df9sA7+OKnq0qlz/GaQg==", + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.4.15", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", + "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@langchain/community": { + "version": "0.2.12", + "resolved": "https://registry.npmjs.org/@langchain/community/-/community-0.2.12.tgz", + "integrity": "sha512-bFhWNyPXqd/V2fduJ6BWipEkAQQHLnGs6IqKmxJMf7FvhN16DfhWK/PpPrNUltE/b5cl5gDuTzdpEsyvuw7fHg==", + "dependencies": { + "@langchain/core": "~0.2.6", + "@langchain/openai": "~0.1.0", + "binary-extensions": "^2.2.0", + "expr-eval": "^2.0.2", + "flat": "^5.0.2", + "js-yaml": "^4.1.0", + "langchain": "0.2.3", + "langsmith": "~0.1.30", + "uuid": "^9.0.0", + "zod": "^3.22.3", + "zod-to-json-schema": "^3.22.5" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@aws-crypto/sha256-js": "^5.0.0", + "@aws-sdk/client-bedrock-agent-runtime": "^3.583.0", + "@aws-sdk/client-bedrock-runtime": "^3.422.0", + "@aws-sdk/client-dynamodb": "^3.310.0", + "@aws-sdk/client-kendra": "^3.352.0", + "@aws-sdk/client-lambda": "^3.310.0", + "@aws-sdk/client-s3": "^3.310.0", + "@aws-sdk/client-sagemaker-runtime": "^3.310.0", + "@aws-sdk/client-sfn": "^3.310.0", + "@aws-sdk/credential-provider-node": "^3.388.0", + "@azure/search-documents": "^12.0.0", + "@azure/storage-blob": "^12.15.0", + "@browserbasehq/sdk": "*", + "@clickhouse/client": "^0.2.5", + "@cloudflare/ai": "*", + "@datastax/astra-db-ts": "^1.0.0", + "@elastic/elasticsearch": "^8.4.0", + "@getmetal/metal-sdk": "*", + "@getzep/zep-cloud": "*", + "@getzep/zep-js": "^0.9.0", + "@gomomento/sdk": "^1.51.1", + "@gomomento/sdk-core": "^1.51.1", + "@google-ai/generativelanguage": "*", + "@google-cloud/storage": "^6.10.1 || ^7.7.0", + "@gradientai/nodejs-sdk": "^1.2.0", + "@huggingface/inference": "^2.6.4", + "@layerup/layerup-security": "^1.5.12", + "@mendable/firecrawl-js": "^0.0.13", + "@mlc-ai/web-llm": "0.2.46", + "@mozilla/readability": "*", + "@neondatabase/serverless": "*", + "@notionhq/client": "^2.2.10", + "@opensearch-project/opensearch": "*", + "@pinecone-database/pinecone": "*", + "@planetscale/database": "^1.8.0", + "@premai/prem-sdk": "^0.3.25", + "@qdrant/js-client-rest": "^1.8.2", + "@raycast/api": "^1.55.2", + "@rockset/client": "^0.9.1", + "@smithy/eventstream-codec": "^2.0.5", + "@smithy/protocol-http": "^3.0.6", + "@smithy/signature-v4": "^2.0.10", + "@smithy/util-utf8": "^2.0.0", + "@spider-cloud/spider-client": "^0.0.21", + "@supabase/postgrest-js": "^1.1.1", + "@supabase/supabase-js": "^2.10.0", + "@tensorflow-models/universal-sentence-encoder": "*", + "@tensorflow/tfjs-converter": "*", + "@tensorflow/tfjs-core": "*", + "@upstash/ratelimit": "^1.1.3", + "@upstash/redis": "^1.20.6", + "@upstash/vector": "^1.1.1", + "@vercel/kv": "^0.2.3", + "@vercel/postgres": "^0.5.0", + "@writerai/writer-sdk": "^0.40.2", + "@xata.io/client": "^0.28.0", + "@xenova/transformers": "^2.5.4", + "@zilliz/milvus2-sdk-node": ">=2.3.5", + "apify-client": "^2.7.1", + "assemblyai": "^4.0.0", + "better-sqlite3": ">=9.4.0 <12.0.0", + "cassandra-driver": "^4.7.2", + "cborg": "^4.1.1", + "cheerio": "^1.0.0-rc.12", + "chromadb": "*", + "closevector-common": "0.1.3", + "closevector-node": "0.1.6", + "closevector-web": "0.1.6", + "cohere-ai": "*", + "convex": "^1.3.1", + "couchbase": "^4.3.0", + "crypto-js": "^4.2.0", + "d3-dsv": "^2.0.0", + "discord.js": "^14.14.1", + "dria": "^0.0.3", + "duck-duck-scrape": "^2.2.5", + "epub2": "^3.0.1", + "faiss-node": "^0.5.1", + "firebase-admin": "^11.9.0 || ^12.0.0", + "google-auth-library": "*", + "googleapis": "^126.0.1", + "hnswlib-node": "^3.0.0", + "html-to-text": "^9.0.5", + "ignore": "^5.2.0", + "interface-datastore": "^8.2.11", + "ioredis": "^5.3.2", + "it-all": "^3.0.4", + "jsdom": "*", + "jsonwebtoken": "^9.0.2", + "llmonitor": "^0.5.9", + "lodash": "^4.17.21", + "lunary": "^0.6.11", + "mammoth": "^1.6.0", + "mongodb": ">=5.2.0", + "mysql2": "^3.3.3", + "neo4j-driver": "*", + "node-llama-cpp": "*", + "notion-to-md": "^3.1.0", + "officeparser": "^4.0.4", + "pdf-parse": "1.1.1", + "pg": "^8.11.0", + "pg-copy-streams": "^6.0.5", + "pickleparser": "^0.2.1", + "playwright": "^1.32.1", + "portkey-ai": "^0.1.11", + "puppeteer": "^19.7.2", + "redis": "*", + "replicate": "^0.29.4", + "sonix-speech-recognition": "^2.1.1", + "srt-parser-2": "^1.2.3", + "typeorm": "^0.3.20", + "typesense": "^1.5.3", + "usearch": "^1.1.1", + "vectordb": "^0.1.4", + "voy-search": "0.6.2", + "weaviate-ts-client": "*", + "web-auth-library": "^1.0.3", + "ws": "^8.14.2", + "youtube-transcript": "^1.0.6", + "youtubei.js": "^9.1.0" + }, + "peerDependenciesMeta": { + "@aws-crypto/sha256-js": { + "optional": true + }, + "@aws-sdk/client-bedrock-agent-runtime": { + "optional": true + }, + "@aws-sdk/client-bedrock-runtime": { + "optional": true + }, + "@aws-sdk/client-dynamodb": { + "optional": true + }, + "@aws-sdk/client-kendra": { + "optional": true + }, + "@aws-sdk/client-lambda": { + "optional": true + }, + "@aws-sdk/client-s3": { + "optional": true + }, + "@aws-sdk/client-sagemaker-runtime": { + "optional": true + }, + "@aws-sdk/client-sfn": { + "optional": true + }, + "@aws-sdk/credential-provider-node": { + "optional": true + }, + "@azure/search-documents": { + "optional": true + }, + "@azure/storage-blob": { + "optional": true + }, + "@browserbasehq/sdk": { + "optional": true + }, + "@clickhouse/client": { + "optional": true + }, + "@cloudflare/ai": { + "optional": true + }, + "@datastax/astra-db-ts": { + "optional": true + }, + "@elastic/elasticsearch": { + "optional": true + }, + "@getmetal/metal-sdk": { + "optional": true + }, + "@getzep/zep-cloud": { + "optional": true + }, + "@getzep/zep-js": { + "optional": true + }, + "@gomomento/sdk": { + "optional": true + }, + "@gomomento/sdk-core": { + "optional": true + }, + "@google-ai/generativelanguage": { + "optional": true + }, + "@google-cloud/storage": { + "optional": true + }, + "@gradientai/nodejs-sdk": { + "optional": true + }, + "@huggingface/inference": { + "optional": true + }, + "@layerup/layerup-security": { + "optional": true + }, + "@mendable/firecrawl-js": { + "optional": true + }, + "@mlc-ai/web-llm": { + "optional": true + }, + "@mozilla/readability": { + "optional": true + }, + "@neondatabase/serverless": { + "optional": true + }, + "@notionhq/client": { + "optional": true + }, + "@opensearch-project/opensearch": { + "optional": true + }, + "@pinecone-database/pinecone": { + "optional": true + }, + "@planetscale/database": { + "optional": true + }, + "@premai/prem-sdk": { + "optional": true + }, + "@qdrant/js-client-rest": { + "optional": true + }, + "@raycast/api": { + "optional": true + }, + "@rockset/client": { + "optional": true + }, + "@smithy/eventstream-codec": { + "optional": true + }, + "@smithy/protocol-http": { + "optional": true + }, + "@smithy/signature-v4": { + "optional": true + }, + "@smithy/util-utf8": { + "optional": true + }, + "@spider-cloud/spider-client": { + "optional": true + }, + "@supabase/postgrest-js": { + "optional": true + }, + "@supabase/supabase-js": { + "optional": true + }, + "@tensorflow-models/universal-sentence-encoder": { + "optional": true + }, + "@tensorflow/tfjs-converter": { + "optional": true + }, + "@tensorflow/tfjs-core": { + "optional": true + }, + "@upstash/ratelimit": { + "optional": true + }, + "@upstash/redis": { + "optional": true + }, + "@upstash/vector": { + "optional": true + }, + "@vercel/kv": { + "optional": true + }, + "@vercel/postgres": { + "optional": true + }, + "@writerai/writer-sdk": { + "optional": true + }, + "@xata.io/client": { + "optional": true + }, + "@xenova/transformers": { + "optional": true + }, + "@zilliz/milvus2-sdk-node": { + "optional": true + }, + "apify-client": { + "optional": true + }, + "assemblyai": { + "optional": true + }, + "better-sqlite3": { + "optional": true + }, + "cassandra-driver": { + "optional": true + }, + "cborg": { + "optional": true + }, + "cheerio": { + "optional": true + }, + "chromadb": { + "optional": true + }, + "closevector-common": { + "optional": true + }, + "closevector-node": { + "optional": true + }, + "closevector-web": { + "optional": true + }, + "cohere-ai": { + "optional": true + }, + "convex": { + "optional": true + }, + "couchbase": { + "optional": true + }, + "crypto-js": { + "optional": true + }, + "d3-dsv": { + "optional": true + }, + "discord.js": { + "optional": true + }, + "dria": { + "optional": true + }, + "duck-duck-scrape": { + "optional": true + }, + "epub2": { + "optional": true + }, + "faiss-node": { + "optional": true + }, + "firebase-admin": { + "optional": true + }, + "google-auth-library": { + "optional": true + }, + "googleapis": { + "optional": true + }, + "hnswlib-node": { + "optional": true + }, + "html-to-text": { + "optional": true + }, + "ignore": { + "optional": true + }, + "interface-datastore": { + "optional": true + }, + "ioredis": { + "optional": true + }, + "it-all": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "jsonwebtoken": { + "optional": true + }, + "llmonitor": { + "optional": true + }, + "lodash": { + "optional": true + }, + "lunary": { + "optional": true + }, + "mammoth": { + "optional": true + }, + "mongodb": { + "optional": true + }, + "mysql2": { + "optional": true + }, + "neo4j-driver": { + "optional": true + }, + "node-llama-cpp": { + "optional": true + }, + "notion-to-md": { + "optional": true + }, + "officeparser": { + "optional": true + }, + "pdf-parse": { + "optional": true + }, + "pg": { + "optional": true + }, + "pg-copy-streams": { + "optional": true + }, + "pickleparser": { + "optional": true + }, + "playwright": { + "optional": true + }, + "portkey-ai": { + "optional": true + }, + "puppeteer": { + "optional": true + }, + "redis": { + "optional": true + }, + "replicate": { + "optional": true + }, + "sonix-speech-recognition": { + "optional": true + }, + "srt-parser-2": { + "optional": true + }, + "typeorm": { + "optional": true + }, + "typesense": { + "optional": true + }, + "usearch": { + "optional": true + }, + "vectordb": { + "optional": true + }, + "voy-search": { + "optional": true + }, + "weaviate-ts-client": { + "optional": true + }, + "web-auth-library": { + "optional": true + }, + "ws": { + "optional": true + }, + "youtube-transcript": { + "optional": true + }, + "youtubei.js": { + "optional": true + } + } + }, + "node_modules/@langchain/community/node_modules/@types/node": { + "version": "18.19.36", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.36.tgz", + "integrity": "sha512-tX1BNmYSWEvViftB26VLNxT6mEr37M7+ldUtq7rlKnv4/2fKYsJIOmqJAjT6h1DNuwQjIKgw3VJ/Dtw3yiTIQw==", "dependencies": { - "@fortaine/fetch-event-source": "^3.0.6", - "cross-fetch": "^3.1.5" + "undici-types": "~5.26.4" + } + }, + "node_modules/@langchain/community/node_modules/langchain": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.2.3.tgz", + "integrity": "sha512-T9xR7zd+Nj0oXy6WoYKmZLy0DlQiDLFPGYWdOXDxy+AvqlujoPdVQgDSpdqiOHvAjezrByAoKxoHCz5XMwTP/Q==", + "dependencies": { + "@langchain/core": "~0.2.0", + "@langchain/openai": "~0.0.28", + "@langchain/textsplitters": "~0.0.0", + "binary-extensions": "^2.2.0", + "js-tiktoken": "^1.0.12", + "js-yaml": "^4.1.0", + "jsonpointer": "^5.0.1", + "langchainhub": "~0.0.8", + "langsmith": "~0.1.7", + "ml-distance": "^4.0.0", + "openapi-types": "^12.1.3", + "p-retry": "4", + "uuid": "^9.0.0", + "yaml": "^2.2.1", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@aws-sdk/client-s3": "^3.310.0", + "@aws-sdk/client-sagemaker-runtime": "^3.310.0", + "@aws-sdk/client-sfn": "^3.310.0", + "@aws-sdk/credential-provider-node": "^3.388.0", + "@azure/storage-blob": "^12.15.0", + "@browserbasehq/sdk": "*", + "@gomomento/sdk": "^1.51.1", + "@gomomento/sdk-core": "^1.51.1", + "@gomomento/sdk-web": "^1.51.1", + "@mendable/firecrawl-js": "^0.0.13", + "@notionhq/client": "^2.2.10", + "@pinecone-database/pinecone": "*", + "@supabase/supabase-js": "^2.10.0", + "@vercel/kv": "^0.2.3", + "@xata.io/client": "^0.28.0", + "apify-client": "^2.7.1", + "assemblyai": "^4.0.0", + "axios": "*", + "cheerio": "^1.0.0-rc.12", + "chromadb": "*", + "convex": "^1.3.1", + "couchbase": "^4.3.0", + "d3-dsv": "^2.0.0", + "epub2": "^3.0.1", + "fast-xml-parser": "*", + "handlebars": "^4.7.8", + "html-to-text": "^9.0.5", + "ignore": "^5.2.0", + "ioredis": "^5.3.2", + "jsdom": "*", + "mammoth": "^1.6.0", + "mongodb": ">=5.2.0", + "node-llama-cpp": "*", + "notion-to-md": "^3.1.0", + "officeparser": "^4.0.4", + "pdf-parse": "1.1.1", + "peggy": "^3.0.2", + "playwright": "^1.32.1", + "puppeteer": "^19.7.2", + "pyodide": "^0.24.1", + "redis": "^4.6.4", + "sonix-speech-recognition": "^2.1.1", + "srt-parser-2": "^1.2.3", + "typeorm": "^0.3.12", + "weaviate-ts-client": "*", + "web-auth-library": "^1.0.3", + "ws": "^8.14.2", + "youtube-transcript": "^1.0.6", + "youtubei.js": "^9.1.0" + }, + "peerDependenciesMeta": { + "@aws-sdk/client-s3": { + "optional": true + }, + "@aws-sdk/client-sagemaker-runtime": { + "optional": true + }, + "@aws-sdk/client-sfn": { + "optional": true + }, + "@aws-sdk/credential-provider-node": { + "optional": true + }, + "@azure/storage-blob": { + "optional": true + }, + "@browserbasehq/sdk": { + "optional": true + }, + "@gomomento/sdk": { + "optional": true + }, + "@gomomento/sdk-core": { + "optional": true + }, + "@gomomento/sdk-web": { + "optional": true + }, + "@mendable/firecrawl-js": { + "optional": true + }, + "@notionhq/client": { + "optional": true + }, + "@pinecone-database/pinecone": { + "optional": true + }, + "@supabase/supabase-js": { + "optional": true + }, + "@vercel/kv": { + "optional": true + }, + "@xata.io/client": { + "optional": true + }, + "apify-client": { + "optional": true + }, + "assemblyai": { + "optional": true + }, + "axios": { + "optional": true + }, + "cheerio": { + "optional": true + }, + "chromadb": { + "optional": true + }, + "convex": { + "optional": true + }, + "couchbase": { + "optional": true + }, + "d3-dsv": { + "optional": true + }, + "epub2": { + "optional": true + }, + "faiss-node": { + "optional": true + }, + "fast-xml-parser": { + "optional": true + }, + "handlebars": { + "optional": true + }, + "html-to-text": { + "optional": true + }, + "ignore": { + "optional": true + }, + "ioredis": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "mammoth": { + "optional": true + }, + "mongodb": { + "optional": true + }, + "node-llama-cpp": { + "optional": true + }, + "notion-to-md": { + "optional": true + }, + "officeparser": { + "optional": true + }, + "pdf-parse": { + "optional": true + }, + "peggy": { + "optional": true + }, + "playwright": { + "optional": true + }, + "puppeteer": { + "optional": true + }, + "pyodide": { + "optional": true + }, + "redis": { + "optional": true + }, + "sonix-speech-recognition": { + "optional": true + }, + "srt-parser-2": { + "optional": true + }, + "typeorm": { + "optional": true + }, + "weaviate-ts-client": { + "optional": true + }, + "web-auth-library": { + "optional": true + }, + "ws": { + "optional": true + }, + "youtube-transcript": { + "optional": true + }, + "youtubei.js": { + "optional": true + } } }, - "node_modules/@dqbd/tiktoken": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/@dqbd/tiktoken/-/tiktoken-1.0.7.tgz", - "integrity": "sha512-bhR5k5W+8GLzysjk8zTMVygQZsgvf7W1F0IlL4ZQ5ugjo5rCyiwGM5d8DYriXspytfu98tv59niang3/T+FoDw==" - }, - "node_modules/@fortaine/fetch-event-source": { - "version": "3.0.6", - "resolved": "https://registry.npmjs.org/@fortaine/fetch-event-source/-/fetch-event-source-3.0.6.tgz", - "integrity": "sha512-621GAuLMvKtyZQ3IA6nlDWhV1V/7PGOTNIGLUifxt0KzM+dZIweJ6F3XvQF3QnqeNfS1N7WQ0Kil1Di/lhChEw==", + "node_modules/@langchain/community/node_modules/langchain/node_modules/@langchain/openai": { + "version": "0.0.34", + "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.0.34.tgz", + "integrity": "sha512-M+CW4oXle5fdoz2T2SwdOef8pl3/1XmUx1vjn2mXUVM/128aO0l23FMF0SNBsAbRV6P+p/TuzjodchJbi0Ht/A==", + "dependencies": { + "@langchain/core": ">0.1.56 <0.3.0", + "js-tiktoken": "^1.0.12", + "openai": "^4.41.1", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, "engines": { - "node": ">=16.15" + "node": ">=18" } }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", - "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", + "node_modules/@langchain/community/node_modules/openai": { + "version": "4.51.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.51.0.tgz", + "integrity": "sha512-UKuWc3/qQyklqhHM8CbdXCv0Z0obap6T0ECdcO5oATQxAbKE5Ky3YCXFQY207z+eGG6ez4U9wvAcuMygxhmStg==", "dependencies": { - "@jridgewell/set-array": "^1.0.1", - "@jridgewell/sourcemap-codec": "^1.4.10", - "@jridgewell/trace-mapping": "^0.3.9" + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" }, - "engines": { - "node": ">=6.0.0" + "bin": { + "openai": "bin/cli" } }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz", - "integrity": "sha512-F2msla3tad+Mfht5cJq7LSXcdudKTWCVYUgw6pLFOOHSTtZlj6SWNYAp+AhuqLmWdBO2X5hPrLcu8cVP8fy28w==", + "node_modules/@langchain/core": { + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.2.7.tgz", + "integrity": "sha512-FdFiNWhszFuUyAhYdY+l5DtPnAnWCAjXMnkLmUJ1J54NeUiUm7gy26Hnd4bkvaOQJ8ddHH/EX03ZwdoYfLv1jw==", + "dependencies": { + "ansi-styles": "^5.0.0", + "camelcase": "6", + "decamelize": "1.2.0", + "js-tiktoken": "^1.0.12", + "langsmith": "~0.1.30", + "ml-distance": "^4.0.0", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "p-retry": "4", + "uuid": "^9.0.0", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, "engines": { - "node": ">=6.0.0" + "node": ">=18" } }, - "node_modules/@jridgewell/set-array": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", - "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", + "node_modules/@langchain/openai": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.1.3.tgz", + "integrity": "sha512-riv/JC9x2A8b7GcHu8sx+mlZJ8KAwSSi231IPTlcciYnKozmrQ5H0vrtiD31fxiDbaRsk7tyCpkSBIOQEo7CyQ==", + "dependencies": { + "@langchain/core": ">=0.2.5 <0.3.0", + "js-tiktoken": "^1.0.12", + "openai": "^4.49.1", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, "engines": { - "node": ">=6.0.0" + "node": ">=18" } }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==" + "node_modules/@langchain/openai/node_modules/@types/node": { + "version": "18.19.36", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.36.tgz", + "integrity": "sha512-tX1BNmYSWEvViftB26VLNxT6mEr37M7+ldUtq7rlKnv4/2fKYsJIOmqJAjT6h1DNuwQjIKgw3VJ/Dtw3yiTIQw==", + "dependencies": { + "undici-types": "~5.26.4" + } }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.18", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.18.tgz", - "integrity": "sha512-w+niJYzMHdd7USdiH2U6869nqhD2nbfZXND5Yp93qIbEmnDNk7PD48o+YchRVpzMU7M6jVCbenTR7PA1FLQ9pA==", + "node_modules/@langchain/openai/node_modules/openai": { + "version": "4.51.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.51.0.tgz", + "integrity": "sha512-UKuWc3/qQyklqhHM8CbdXCv0Z0obap6T0ECdcO5oATQxAbKE5Ky3YCXFQY207z+eGG6ez4U9wvAcuMygxhmStg==", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7", + "web-streams-polyfill": "^3.2.1" + }, + "bin": { + "openai": "bin/cli" + } + }, + "node_modules/@langchain/pinecone": { + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/@langchain/pinecone/-/pinecone-0.0.7.tgz", + "integrity": "sha512-PQztQOjMyVSBz4AofpKMn619PyVVF6Gl0OJzubolZNUA4lbR3UR+kENEe81WD6RSbBy3K9brOfjDDYx1hmtqOw==", "dependencies": { - "@jridgewell/resolve-uri": "3.1.0", - "@jridgewell/sourcemap-codec": "1.4.14" + "@langchain/core": ">0.2.0 <0.3.0", + "@pinecone-database/pinecone": "^2.2.0", + "flat": "^5.0.2", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=18" } }, - "node_modules/@jridgewell/trace-mapping/node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.14", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.14.tgz", - "integrity": "sha512-XPSJHWmi394fuUuzDnGz1wiKqWfo1yXecHQMRf2l6hztTO+nPru658AyDngaBe7isIxEkRsPR3FZh+s7iVa4Uw==" + "node_modules/@langchain/textsplitters": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@langchain/textsplitters/-/textsplitters-0.0.3.tgz", + "integrity": "sha512-cXWgKE3sdWLSqAa8ykbCcUsUF1Kyr5J3HOWYGuobhPEycXW4WI++d5DhzdpL238mzoEXTi90VqfSCra37l5YqA==", + "dependencies": { + "@langchain/core": ">0.2.0 <0.3.0", + "js-tiktoken": "^1.0.12" + }, + "engines": { + "node": ">=18" + } }, "node_modules/@next/env": { "version": "13.4.1", @@ -292,69 +1156,98 @@ } }, "node_modules/@pinecone-database/pinecone": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-0.1.5.tgz", - "integrity": "sha512-Amkc6JFjhRzKqf5GkdI2Pd4H7+EqUtNp7kaJdCSgDT2PG2cokx+bGyRBWzrkYD7Kwo5jtWaPNWcblysEUPWCVg==", + "version": "2.2.2", + "resolved": "https://registry.npmjs.org/@pinecone-database/pinecone/-/pinecone-2.2.2.tgz", + "integrity": "sha512-gbe/4SowHc64pHIm0kBdgY9hVdzsQnnnpcWviwYMB33gOmsL8brvE8fUSpl1dLDvdyXzKcQkzdBsjCDlqgpdMA==", "dependencies": { - "cross-fetch": "^3.1.5" + "@sinclair/typebox": "^0.29.0", + "ajv": "^8.12.0", + "cross-fetch": "^3.1.5", + "encoding": "^0.1.13" }, "engines": { "node": ">=14.0.0" } }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "optional": true, + "engines": { + "node": ">=14" + } + }, + "node_modules/@sinclair/typebox": { + "version": "0.29.6", + "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.29.6.tgz", + "integrity": "sha512-aX5IFYWlMa7tQ8xZr3b2gtVReCvg7f3LEhjir/JAjX2bJCMVJA5tIPv30wTD4KDfcwMd7DDYY3hFDeGmOgtrZQ==" + }, + "node_modules/@supabase/auth-js": { + "version": "2.64.2", + "resolved": "https://registry.npmjs.org/@supabase/auth-js/-/auth-js-2.64.2.tgz", + "integrity": "sha512-s+lkHEdGiczDrzXJ1YWt2y3bxRi+qIUnXcgkpLSrId7yjBeaXBFygNjTaoZLG02KNcYwbuZ9qkEIqmj2hF7svw==", + "dependencies": { + "@supabase/node-fetch": "^2.6.14" + } + }, "node_modules/@supabase/functions-js": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@supabase/functions-js/-/functions-js-2.1.1.tgz", - "integrity": "sha512-bIR1Puae6W+1/MzPfYBWOG/SCWGo4B5CB7c0ZZksvliNEAzhxNBJ0UFKYINcGdGtxG8ZC+1xr3utWpNZNwnoRw==", + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/@supabase/functions-js/-/functions-js-2.4.1.tgz", + "integrity": "sha512-8sZ2ibwHlf+WkHDUZJUXqqmPvWQ3UHN0W30behOJngVh/qHHekhJLCFbh0AjkE9/FqqXtf9eoVvmYgfCLk5tNA==", "dependencies": { - "cross-fetch": "^3.1.5" + "@supabase/node-fetch": "^2.6.14" } }, - "node_modules/@supabase/gotrue-js": { - "version": "2.29.0", - "resolved": "https://registry.npmjs.org/@supabase/gotrue-js/-/gotrue-js-2.29.0.tgz", - "integrity": "sha512-QJepUxSXpgcyMhDtEusRyGtCcYSqy4wRDf3BQGqLUDaU/sRRclO07NCHW8nBqGW6KZZ6oNLfKX2AQz621dmIPw==", + "node_modules/@supabase/node-fetch": { + "version": "2.6.15", + "resolved": "https://registry.npmjs.org/@supabase/node-fetch/-/node-fetch-2.6.15.tgz", + "integrity": "sha512-1ibVeYUacxWYi9i0cf5efil6adJ9WRyZBLivgjs+AUpewx1F3xPi7gLgaASI2SmIQxPoCEjAsLAzKPgMJVgOUQ==", "dependencies": { - "cross-fetch": "^3.1.5" + "whatwg-url": "^5.0.0" + }, + "engines": { + "node": "4.x || >=6.0.0" } }, "node_modules/@supabase/postgrest-js": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-1.7.0.tgz", - "integrity": "sha512-wLADHZ5jm7LljF4GigK0H2vc1wGupBY2hGYfb4fVo0UuyMftmA6tOYy+ZpMH/vPq01CUFwXGwvIke6kyqh/QDg==", + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/@supabase/postgrest-js/-/postgrest-js-1.15.5.tgz", + "integrity": "sha512-YR4TiitTE2hizT7mB99Cl3V9i00RAY5sUxS2/NuWWzkreM7OeYlP2OqnqVwwb4z6ILn+j8x9e/igJDepFhjswQ==", "dependencies": { - "cross-fetch": "^3.1.5" + "@supabase/node-fetch": "^2.6.14" } }, "node_modules/@supabase/realtime-js": { - "version": "2.7.2", - "resolved": "https://registry.npmjs.org/@supabase/realtime-js/-/realtime-js-2.7.2.tgz", - "integrity": "sha512-Fi6xAl5PUkqnjl3wo4rdcQIbMG3+yTRX1aUZe/yfvTG84RMvmCXJ1yN6MmafVLeZpU1xkaz5Vx4L0tnHcLiy6w==", + "version": "2.9.5", + "resolved": "https://registry.npmjs.org/@supabase/realtime-js/-/realtime-js-2.9.5.tgz", + "integrity": "sha512-TEHlGwNGGmKPdeMtca1lFTYCedrhTAv3nZVoSjrKQ+wkMmaERuCe57zkC5KSWFzLYkb5FVHW8Hrr+PX1DDwplQ==", "dependencies": { + "@supabase/node-fetch": "^2.6.14", "@types/phoenix": "^1.5.4", - "@types/websocket": "^1.0.3", - "websocket": "^1.0.34" + "@types/ws": "^8.5.10", + "ws": "^8.14.2" } }, "node_modules/@supabase/storage-js": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.5.1.tgz", - "integrity": "sha512-nkR0fQA9ScAtIKA3vNoPEqbZv1k5B5HVRYEvRWdlP6mUpFphM9TwPL2jZ/ztNGMTG5xT6SrHr+H7Ykz8qzbhjw==", + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/@supabase/storage-js/-/storage-js-2.6.0.tgz", + "integrity": "sha512-REAxr7myf+3utMkI2oOmZ6sdplMZZ71/2NEIEMBZHL9Fkmm3/JnaOZVSRqvG4LStYj2v5WhCruCzuMn6oD/Drw==", "dependencies": { - "cross-fetch": "^3.1.5" + "@supabase/node-fetch": "^2.6.14" } }, "node_modules/@supabase/supabase-js": { - "version": "2.24.0", - "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.24.0.tgz", - "integrity": "sha512-zrAm+hp6DBICqZ7xVPk+KofmlfjJWQzXuf2sHAyPz8XVjpha84z2OVWcow2aI10YkMOrPwhRtBBQYJOnh/fx2w==", + "version": "2.43.5", + "resolved": "https://registry.npmjs.org/@supabase/supabase-js/-/supabase-js-2.43.5.tgz", + "integrity": "sha512-Y4GukjZWW6ouohMaPlYz8tSz9ykf9jY7w9/RhqKuScmla3Xiklce8eLr8TYAtA+oQYCWxo3RgS3B6O4rd/72FA==", "dependencies": { - "@supabase/functions-js": "^2.1.0", - "@supabase/gotrue-js": "^2.26.0", - "@supabase/postgrest-js": "^1.7.0", - "@supabase/realtime-js": "^2.7.2", - "@supabase/storage-js": "^2.5.1", - "cross-fetch": "^3.1.5" + "@supabase/auth-js": "2.64.2", + "@supabase/functions-js": "2.4.1", + "@supabase/node-fetch": "2.6.15", + "@supabase/postgrest-js": "1.15.5", + "@supabase/realtime-js": "2.9.5", + "@supabase/storage-js": "2.6.0" } }, "node_modules/@swc/helpers": { @@ -366,9 +1259,9 @@ } }, "node_modules/@types/cors": { - "version": "2.8.13", - "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.13.tgz", - "integrity": "sha512-RG8AStHlUiV5ysZQKq97copd2UmVYw3/pRMLefISZ3S1hK104Cwm7iLQ3fTKx+lsUH2CE8FlLaYeEA2LSeqYUA==", + "version": "2.8.17", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.17.tgz", + "integrity": "sha512-8CGDvrBj1zgo2qE+oS3pOCyYNqCPryMWY2bGfwA0dcfopWGgxs+78df0Rs3rc9THP4JkOhLsAa+15VdpAqkcUA==", "dev": true, "dependencies": { "@types/node": "*" @@ -379,15 +1272,24 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-20.1.0.tgz", "integrity": "sha512-O+z53uwx64xY7D6roOi4+jApDGFg0qn6WHcxe5QeqjMaTezBO/mxdfFXIVAVVyNWKx84OmPB3L8kbVYOTeN34A==" }, + "node_modules/@types/node-fetch": { + "version": "2.6.11", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.11.tgz", + "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", + "dependencies": { + "@types/node": "*", + "form-data": "^4.0.0" + } + }, "node_modules/@types/phoenix": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@types/phoenix/-/phoenix-1.6.0.tgz", - "integrity": "sha512-qwfpsHmFuhAS/dVd4uBIraMxRd56vwBUYQGZ6GpXnFuM2XMRFJbIyruFKKlW2daQliuYZwe0qfn/UjFCDKic5g==" + "version": "1.6.5", + "resolved": "https://registry.npmjs.org/@types/phoenix/-/phoenix-1.6.5.tgz", + "integrity": "sha512-xegpDuR+z0UqG9fwHqNoy3rI7JDlvaPh2TY47Fl80oq6g+hXT+c/LEuE43X48clZ6lOfANl5WrPur9fYO1RJ/w==" }, "node_modules/@types/prop-types": { - "version": "15.7.5", - "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", - "integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==", + "version": "15.7.12", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.12.tgz", + "integrity": "sha512-5zvhXYtRNRluoE/jAp4GVsSduVUzNWKkOZrCDBWYtE7biZywwdC2AcEzg+cSMLFRfVgeAFqpfNabiPjxFddV1Q==", "dev": true }, "node_modules/@types/react": { @@ -407,19 +1309,61 @@ "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==" }, "node_modules/@types/scheduler": { - "version": "0.16.3", - "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.3.tgz", - "integrity": "sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ==", + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.23.0.tgz", + "integrity": "sha512-YIoDCTH3Af6XM5VuwGG/QL/CJqga1Zm3NkU3HZ4ZHK2fRMPYP1VczsTUqtsf43PH/iJNVlPHAo2oWX7BSdB2Hw==", "dev": true }, - "node_modules/@types/websocket": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@types/websocket/-/websocket-1.0.5.tgz", - "integrity": "sha512-NbsqiNX9CnEfC1Z0Vf4mE1SgAJ07JnRYcNex7AJ9zAVzmiGHmjKFEk7O4TJIsgv2B1sLEb6owKFZrACwdYngsQ==", + "node_modules/@types/uuid": { + "version": "9.0.8", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.8.tgz", + "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==" + }, + "node_modules/@types/ws": { + "version": "8.5.10", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-8.5.10.tgz", + "integrity": "sha512-vmQSUcfalpIq0R9q7uTo2lXs6eGIpt9wtnLdMv9LVpIjCA/+ufZRozlVoVelIYixx1ugCBKDhn89vnsEGOCx9A==", "dependencies": { "@types/node": "*" } }, + "node_modules/abort-controller": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "^5.0.0" + }, + "engines": { + "node": ">=6.5" + } + }, + "node_modules/agentkeepalive": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.5.0.tgz", + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", + "dependencies": { + "humanize-ms": "^1.2.1" + }, + "engines": { + "node": ">= 8.0.0" + } + }, + "node_modules/ajv": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.16.0.tgz", + "integrity": "sha512-F0twR8U1ZU67JIEtekUcLkXkoO5mMMmgGD8sK/xUFzJ805jxHQl92hImFAqqXMyMYjSPOyUPAwHYhB72g5sTXw==", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", + "uri-js": "^4.4.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, "node_modules/amdefine": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/amdefine/-/amdefine-1.0.1.tgz", @@ -429,6 +1373,17 @@ "node": ">=0.4.2" } }, + "node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, "node_modules/ansi-styles": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", @@ -462,6 +1417,11 @@ "resolved": "https://registry.npmjs.org/arg/-/arg-5.0.2.tgz", "integrity": "sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==" }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==" + }, "node_modules/ast-transform": { "version": "0.0.0", "resolved": "https://registry.npmjs.org/ast-transform/-/ast-transform-0.0.0.tgz", @@ -518,11 +1478,11 @@ } }, "node_modules/axios": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.4.0.tgz", - "integrity": "sha512-S4XCWMEmzvo64T9GfvQDOXgYRDJ/wsSZc7Jvdgx5u1sd0JwsuPLqb3SYmusag+edF6ziyMensPVqLTSc1PiSEA==", + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", + "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", "dependencies": { - "follow-redirects": "^1.15.0", + "follow-redirects": "^1.15.6", "form-data": "^4.0.0", "proxy-from-env": "^1.1.0" } @@ -532,12 +1492,34 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/binary-extensions": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", - "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", "engines": { "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/binary-search": { @@ -545,44 +1527,30 @@ "resolved": "https://registry.npmjs.org/binary-search/-/binary-search-1.3.6.tgz", "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==" }, - "node_modules/bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "dependencies": { - "file-uri-to-path": "1.0.0" - } - }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" }, "node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "balanced-match": "^1.0.0" } }, "node_modules/braces": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.2.tgz", - "integrity": "sha512-b8um+L1RzM3WDSzvhm6gIz1yfTbBt6YTlcEKAvsmqCZZFw46z626lVj9j1yEPW33H5H+lBQpZMP1k8l+78Ha0A==", + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dependencies": { - "fill-range": "^7.0.1" + "fill-range": "^7.1.1" }, "engines": { "node": ">=8" } }, - "node_modules/browser-or-node": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/browser-or-node/-/browser-or-node-2.1.1.tgz", - "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==" - }, "node_modules/browser-resolve": { "version": "1.11.3", "resolved": "https://registry.npmjs.org/browser-resolve/-/browser-resolve-1.11.3.tgz", @@ -591,11 +1559,6 @@ "resolve": "1.1.7" } }, - "node_modules/browser-resolve/node_modules/resolve": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz", - "integrity": "sha512-9znBF0vBcaSN3W2j7wKvdERPwqTxSpCq+if5C0WoTCyV9n24rua28jeuQ2pL/HOf+yUe/Mef+H/5p60K0Id3bg==" - }, "node_modules/browserify-optional": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/browserify-optional/-/browserify-optional-1.0.1.tgz", @@ -607,9 +1570,9 @@ } }, "node_modules/browserslist": { - "version": "4.21.5", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.21.5.tgz", - "integrity": "sha512-tUkiguQGW7S3IhB7N+c2MV/HZPSCPAAiYBZXLsBhFB/PCy6ZKKsZrmBayHV9fdGV/ARIfJ14NkxKzRDjvp7L6w==", + "version": "4.23.1", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.1.tgz", + "integrity": "sha512-TUfofFo/KsK/bWZ9TWQ5O26tsWW4Uhmt8IYklbnUa70udB6P2wA7w7o4PY4muaEPBQaAX+CEnmmIA41NVHtPVw==", "funding": [ { "type": "opencollective", @@ -618,13 +1581,17 @@ { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" } ], "dependencies": { - "caniuse-lite": "^1.0.30001449", - "electron-to-chromium": "^1.4.284", - "node-releases": "^2.0.8", - "update-browserslist-db": "^1.0.10" + "caniuse-lite": "^1.0.30001629", + "electron-to-chromium": "^1.4.796", + "node-releases": "^2.0.14", + "update-browserslist-db": "^1.0.16" }, "bin": { "browserslist": "cli.js" @@ -633,18 +1600,6 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, - "node_modules/bufferutil": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/bufferutil/-/bufferutil-4.0.7.tgz", - "integrity": "sha512-kukuqc39WOHtdxtw4UScxF/WVnMFVSQVKhtx3AjZJzhd0RGZZldcrfSEbVsWWe6KNH253574cq5F+wpv0G9pJw==", - "hasInstallScript": true, - "dependencies": { - "node-gyp-build": "^4.3.0" - }, - "engines": { - "node": ">=6.14.2" - } - }, "node_modules/busboy": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/busboy/-/busboy-1.6.0.tgz", @@ -656,6 +1611,17 @@ "node": ">=10.16.0" } }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/camelcase-css": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/camelcase-css/-/camelcase-css-2.0.1.tgz", @@ -665,9 +1631,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001486", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001486.tgz", - "integrity": "sha512-uv7/gXuHi10Whlj0pp5q/tsK/32J2QSqVRKQhs2j8VsDCjgyruAh/eEXHF822VqO9yT6iZKw3nRwZRSPBE9OQg==", + "version": "1.0.30001636", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001636.tgz", + "integrity": "sha512-bMg2vmr8XBsbL6Lr0UHXy/21m84FTxDLWn2FSqMd5PrlbMxwJlQnC2YWYxVgp66PZE+BBNF2jYQUBKCo1FDeZg==", "funding": [ { "type": "opencollective", @@ -683,11 +1649,6 @@ } ] }, - "node_modules/centra": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/centra/-/centra-2.6.0.tgz", - "integrity": "sha512-dgh+YleemrT8u85QL11Z6tYhegAs3MMxsaWAq/oXeAmYJ7VxL3SI9TZtnfaEvNDMAPolj25FXIb3S+HCI4wQaQ==" - }, "node_modules/cheerio": { "version": "1.0.0-rc.12", "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", @@ -725,15 +1686,9 @@ } }, "node_modules/chokidar": { - "version": "3.5.3", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", - "integrity": "sha512-Dr3sfKRP6oTcjf2JmUmFJfeVMvXBdegxB0iVQ5eb2V10uFJUCAS8OByZdVAyVb8xXNz3GjjTgj9kLWsZTqE6kw==", - "funding": [ - { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - ], + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", @@ -746,6 +1701,9 @@ "engines": { "node": ">= 8.10.0" }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, "optionalDependencies": { "fsevents": "~2.3.2" } @@ -762,15 +1720,31 @@ } }, "node_modules/classnames": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz", - "integrity": "sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw==" + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz", + "integrity": "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow==" }, "node_modules/client-only": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/client-only/-/client-only-0.0.1.tgz", "integrity": "sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA==" }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -782,18 +1756,13 @@ "node": ">= 0.8" } }, - "node_modules/commander": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", - "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", - "engines": { - "node": ">= 6" - } - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==" + "node_modules/commander": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-10.0.1.tgz", + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "engines": { + "node": ">=14" + } }, "node_modules/cors": { "version": "2.8.5", @@ -808,11 +1777,24 @@ } }, "node_modules/cross-fetch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", - "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.8.tgz", + "integrity": "sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==", "dependencies": { - "node-fetch": "2.6.7" + "node-fetch": "^2.6.12" + } + }, + "node_modules/cross-spawn": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", + "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "dependencies": { + "path-key": "^3.1.0", + "shebang-command": "^2.0.0", + "which": "^2.0.1" + }, + "engines": { + "node": ">= 8" } }, "node_modules/css-select": { @@ -853,24 +1835,15 @@ } }, "node_modules/csstype": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", - "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==", + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", "dev": true }, - "node_modules/d": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/d/-/d-1.0.1.tgz", - "integrity": "sha512-m62ShEObQ39CfralilEQRjH6oAMtNCV1xJyEx5LpRYUVN+EviphDgUc/F3hnYbADmkiNs67Y+3ylmlG7Lnu+FA==", - "dependencies": { - "es5-ext": "^0.10.50", - "type": "^1.0.1" - } - }, "node_modules/debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", + "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", "dependencies": { "ms": "2.1.2" }, @@ -883,6 +1856,14 @@ } } }, + "node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -953,17 +1934,38 @@ } }, "node_modules/dotenv": { - "version": "16.0.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.0.3.tgz", - "integrity": "sha512-7GO6HghkA5fYG9TYnNxi14/7K9f5occMlp3zXAuSxn7CKCxt9xbNWG7yF8hTCSUchlfWSe3uLmlPfigevRItzQ==", + "version": "16.4.5", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.5.tgz", + "integrity": "sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg==", "engines": { "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" + }, "node_modules/electron-to-chromium": { - "version": "1.4.385", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.385.tgz", - "integrity": "sha512-L9zlje9bIw0h+CwPQumiuVlfMcV4boxRjFIWDcLfFqTZNbkwOExBzfmswytHawObQX4OUhtNv8gIiB21kOurIg==" + "version": "1.4.805", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.805.tgz", + "integrity": "sha512-8W4UJwX/w9T0QSzINJckTKG6CYpAUTqsaWcWIsdud3I1FYJcMgW9QqT1/4CBff/pP/TihWh13OmiyY8neto6vw==" + }, + "node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" + }, + "node_modules/encoding": { + "version": "0.1.13", + "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", + "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", + "dependencies": { + "iconv-lite": "^0.6.2" + } }, "node_modules/entities": { "version": "4.5.0", @@ -976,43 +1978,10 @@ "url": "https://github.com/fb55/entities?sponsor=1" } }, - "node_modules/es5-ext": { - "version": "0.10.62", - "resolved": "https://registry.npmjs.org/es5-ext/-/es5-ext-0.10.62.tgz", - "integrity": "sha512-BHLqn0klhEpnOKSrzn/Xsz2UIW8j+cGmo9JLzr8BiUapV8hPL9+FliFqjwr9ngW7jWdnxv6eO+/LqyhJVqgrjA==", - "hasInstallScript": true, - "dependencies": { - "es6-iterator": "^2.0.3", - "es6-symbol": "^3.1.3", - "next-tick": "^1.1.0" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/es6-iterator": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/es6-iterator/-/es6-iterator-2.0.3.tgz", - "integrity": "sha512-zw4SRzoUkd+cl+ZoE15A9o1oQd920Bb0iOJMQkQhl3jNc03YqVjAhG7scf9C5KWRU/R13Orf588uCC6525o02g==", - "dependencies": { - "d": "1", - "es5-ext": "^0.10.35", - "es6-symbol": "^3.1.1" - } - }, - "node_modules/es6-symbol": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz", - "integrity": "sha512-NJ6Yn3FuDinBaBRWl/q5X/s4koRHBrgKAu+yGI6JCBeiu3qrcbJhwT2GeR/EXVfylRk8dpQVJoLEFhK+Mu31NA==", - "dependencies": { - "d": "^1.0.1", - "ext": "^1.1.2" - } - }, "node_modules/escalade": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", - "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz", + "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==", "engines": { "node": ">=6" } @@ -1070,6 +2039,14 @@ "resolved": "https://registry.npmjs.org/eve/-/eve-0.5.4.tgz", "integrity": "sha512-aqprQ9MAOh1t66PrHxDFmMXPlgNO6Uv1uqvxmwjprQV50jaQ2RqO7O1neY4PJwC+hMnkyMDphu2AQPOPZdjQog==" }, + "node_modules/event-target-shim": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "engines": { + "node": ">=6" + } + }, "node_modules/eventemitter3": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", @@ -1086,23 +2063,15 @@ "integrity": "sha512-DJF0nofFGq0IXJLGq95hfrryP3ZprVAVpyZUnmAk6QhHnm7zCzsHBNFP0i4FKFo2XjOf+JiYUKjT7jQhIeljpg==", "dev": true }, - "node_modules/ext": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/ext/-/ext-1.7.0.tgz", - "integrity": "sha512-6hxeJYaL110a9b5TEJSj0gojyHQAmA2ch5Os+ySCiA1QGdS697XWY1pzsrSjqA9LDEEgdB/KypIlR59RcLuHYw==", - "dependencies": { - "type": "^2.7.2" - } - }, - "node_modules/ext/node_modules/type": { - "version": "2.7.2", - "resolved": "https://registry.npmjs.org/type/-/type-2.7.2.tgz", - "integrity": "sha512-dzlvlNlt6AXU7EBSfpAscydQ7gXB+pPGsPnfJnZpiNJBDj7IaJzQlBZYGdEi4R9HmPdBv2XmWJ6YUtoTa7lmCw==" + "node_modules/fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==" }, "node_modules/fast-glob": { - "version": "3.2.12", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.12.tgz", - "integrity": "sha512-DVj4CQIYYow0BlaelwK1pHl5n5cRSJfM60UA0zK891sVInoPri2Ekj7+e1CT3/3qxXenpI+nBBmQAcJPJgaj4w==", + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", + "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", "dependencies": { "@nodelib/fs.stat": "^2.0.2", "@nodelib/fs.walk": "^1.2.3", @@ -1126,22 +2095,17 @@ } }, "node_modules/fastq": { - "version": "1.15.0", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", - "integrity": "sha512-wBrocU2LCXXa+lWBt8RoIRD89Fi8OdABODa/kEnyeyjS5aZO5/GNvI5sEINADqP/h8M29UHTHUb53sUu5Ihqdw==", + "version": "1.17.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", + "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", "dependencies": { "reusify": "^1.0.4" } }, - "node_modules/file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" - }, "node_modules/fill-range": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", - "integrity": "sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", "dependencies": { "to-regex-range": "^5.0.1" }, @@ -1158,9 +2122,9 @@ } }, "node_modules/follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", @@ -1176,6 +2140,21 @@ } } }, + "node_modules/foreground-child": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.2.1.tgz", + "integrity": "sha512-PXUUyLqrR2XCWICfv6ukppP96sdFwWbNEnfEMt7jNsISjMsvaLNinAHNDYyvkyU+SZG2BTSbT5NjG+vZslfGTA==", + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", @@ -1189,27 +2168,47 @@ "node": ">= 6" } }, + "node_modules/form-data-encoder": { + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==" + }, + "node_modules/formdata-node": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dependencies": { + "node-domexception": "1.0.0", + "web-streams-polyfill": "4.0.0-beta.3" + }, + "engines": { + "node": ">= 12.20" + } + }, + "node_modules/formdata-node/node_modules/web-streams-polyfill": { + "version": "4.0.0-beta.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "engines": { + "node": ">= 14" + } + }, "node_modules/fraction.js": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.2.0.tgz", - "integrity": "sha512-MhLuK+2gUcnZe8ZHlaaINnQLl0xRIGRfcGk2yl8xoQAfHrSsL3rYu6FCmBdkdbhc9EPlwyGHewaRsvwRMJtAlA==", + "version": "4.3.7", + "resolved": "https://registry.npmjs.org/fraction.js/-/fraction.js-4.3.7.tgz", + "integrity": "sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew==", "engines": { "node": "*" }, "funding": { "type": "patreon", - "url": "https://www.patreon.com/infusion" + "url": "https://github.com/sponsors/rawify" } }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==" - }, "node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", "hasInstallScript": true, "optional": true, "os": [ @@ -1220,24 +2219,29 @@ } }, "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.4.1.tgz", + "integrity": "sha512-2jelhlq3E4ho74ZyVLN03oKdAZVUa6UDZzFLVH1H7dnoax+y9qyaq8zBkfDIggjniU19z0wU18y16jMB2eyVIw==", "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "foreground-child": "^3.1.0", + "jackspeak": "^3.1.2", + "minimatch": "^9.0.4", + "minipass": "^7.1.2", + "path-scurry": "^1.11.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" }, "engines": { - "node": "*" + "node": ">=16 || 14 >=14.18" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -1254,25 +2258,15 @@ "node": ">=10.13.0" } }, - "node_modules/has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", "dependencies": { - "function-bind": "^1.1.1" + "function-bind": "^1.1.2" }, "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/hnswlib-node": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/hnswlib-node/-/hnswlib-node-1.4.2.tgz", - "integrity": "sha512-76PIzOaNcX8kOpKwlFPl07uelpctqDMzbiC+Qsk2JWNVkzeU/6iXRk4tfE9z3DoK1RCBrOaFXmQ6RFb1BVF9LA==", - "hasInstallScript": true, - "dependencies": { - "bindings": "^1.5.0", - "node-addon-api": "^6.0.0" + "node": ">= 0.4" } }, "node_modules/htmlparser2": { @@ -1293,19 +2287,24 @@ "entities": "^4.4.0" } }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "node_modules/humanize-ms": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", "dependencies": { - "once": "^1.3.0", - "wrappy": "1" + "ms": "^2.0.0" } }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } }, "node_modules/is-any-array": { "version": "2.0.1", @@ -1324,11 +2323,11 @@ } }, "node_modules/is-core-module": { - "version": "2.12.0", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.0.tgz", - "integrity": "sha512-RECHCBCd/viahWmwj6enj19sKbHfJrddi/6cBDsNTKbNq0f7VeaUkBo60BqzvPqo/W54ChS62Z5qyun7cfOMqQ==", + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", "dependencies": { - "has": "^1.0.3" + "hasown": "^2.0.0" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -1342,6 +2341,14 @@ "node": ">=0.10.0" } }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -1361,24 +2368,65 @@ "node": ">=0.12.0" } }, - "node_modules/is-typedarray": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz", - "integrity": "sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA==" + "node_modules/isexe": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", + "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" + }, + "node_modules/jackspeak": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-3.4.0.tgz", + "integrity": "sha512-JVYhQnN59LVPFCEcVa2C3CrEKYacvjRfqIQl+h8oi91aLYQVWRYbxjPcv1bUiUy/kLmQaANrYfNMCO3kuEDHfw==", + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } }, "node_modules/jiti": { - "version": "1.18.2", - "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.18.2.tgz", - "integrity": "sha512-QAdOptna2NYiSSpv0O/BwoHBSmz4YhpzJHyi+fnMRTXFjp7B8i/YG5Z8IfusxB1ufjcD2Sre1F3R+nX3fvy7gg==", + "version": "1.21.6", + "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.6.tgz", + "integrity": "sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==", "bin": { "jiti": "bin/jiti.js" } }, + "node_modules/js-tiktoken": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.12.tgz", + "integrity": "sha512-L7wURW1fH9Qaext0VzaUDpFGVQgjkdE3Dgsy9/+yXyGEpBKnylTd0mU0bfbNkKDlXRb6TEsZkwuflu1B8uQbJQ==", + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, + "node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" + }, "node_modules/jsonpointer": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/jsonpointer/-/jsonpointer-5.0.1.tgz", @@ -1388,84 +2436,113 @@ } }, "node_modules/langchain": { - "version": "0.0.75", - "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.0.75.tgz", - "integrity": "sha512-5bulBe3pEDnCuO3W84E+pmns6+7oKdNomsApPD1PqjplEwT3szSYaY8bvqJVxYz2Hq1wcS3sL1OyL9rbC/s0NA==", + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-0.2.5.tgz", + "integrity": "sha512-H5WL0NanCdQ+tzoeEt7Fyz9YGdR3wbfDvfQrJvxAO95istKo5JraRh24dzyvqxM9439xwRMNaMIpMwsyqtWDtQ==", "dependencies": { - "@anthropic-ai/sdk": "^0.4.3", - "@dqbd/tiktoken": "^1.0.7", - "ansi-styles": "^5.0.0", + "@langchain/core": "~0.2.0", + "@langchain/openai": "~0.1.0", + "@langchain/textsplitters": "~0.0.0", "binary-extensions": "^2.2.0", - "browser-or-node": "^2.1.1", - "expr-eval": "^2.0.2", - "flat": "^5.0.2", + "js-tiktoken": "^1.0.12", + "js-yaml": "^4.1.0", "jsonpointer": "^5.0.1", + "langchainhub": "~0.0.8", + "langsmith": "~0.1.30", "ml-distance": "^4.0.0", - "object-hash": "^3.0.0", - "openai": "^3.2.0", - "p-queue": "^6.6.2", + "openapi-types": "^12.1.3", "p-retry": "4", "uuid": "^9.0.0", "yaml": "^2.2.1", - "zod": "^3.21.4", - "zod-to-json-schema": "^3.20.4" + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" }, "engines": { "node": ">=18" }, "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.310.0", - "@aws-sdk/client-lambda": "^3.310.0", "@aws-sdk/client-s3": "^3.310.0", - "@clickhouse/client": "^0.0.14", - "@getmetal/metal-sdk": "*", - "@huggingface/inference": "^1.5.1", - "@opensearch-project/opensearch": "*", + "@aws-sdk/client-sagemaker-runtime": "^3.310.0", + "@aws-sdk/client-sfn": "^3.310.0", + "@aws-sdk/credential-provider-node": "^3.388.0", + "@azure/storage-blob": "^12.15.0", + "@browserbasehq/sdk": "*", + "@gomomento/sdk": "^1.51.1", + "@gomomento/sdk-core": "^1.51.1", + "@gomomento/sdk-web": "^1.51.1", + "@mendable/firecrawl-js": "^0.0.13", + "@notionhq/client": "^2.2.10", "@pinecone-database/pinecone": "*", "@supabase/supabase-js": "^2.10.0", - "@tensorflow-models/universal-sentence-encoder": "*", - "@tensorflow/tfjs-converter": "*", - "@tensorflow/tfjs-core": "*", - "@zilliz/milvus2-sdk-node": "^2.2.0", + "@vercel/kv": "^0.2.3", + "@xata.io/client": "^0.28.0", + "apify-client": "^2.7.1", + "assemblyai": "^4.0.0", "axios": "*", "cheerio": "^1.0.0-rc.12", - "chromadb": "^1.4.0", - "cohere-ai": "^5.0.2", + "chromadb": "*", + "convex": "^1.3.1", + "couchbase": "^4.3.0", "d3-dsv": "^2.0.0", "epub2": "^3.0.1", - "hnswlib-node": "^1.4.2", + "fast-xml-parser": "*", + "handlebars": "^4.7.8", "html-to-text": "^9.0.5", - "mammoth": "*", - "mongodb": "^5.2.0", + "ignore": "^5.2.0", + "ioredis": "^5.3.2", + "jsdom": "*", + "mammoth": "^1.6.0", + "mongodb": ">=5.2.0", + "node-llama-cpp": "*", + "notion-to-md": "^3.1.0", + "officeparser": "^4.0.4", "pdf-parse": "1.1.1", + "peggy": "^3.0.2", "playwright": "^1.32.1", "puppeteer": "^19.7.2", + "pyodide": "^0.24.1", "redis": "^4.6.4", - "replicate": "^0.9.0", - "srt-parser-2": "^1.2.2", + "sonix-speech-recognition": "^2.1.1", + "srt-parser-2": "^1.2.3", "typeorm": "^0.3.12", - "weaviate-ts-client": "^1.0.0" + "weaviate-ts-client": "*", + "web-auth-library": "^1.0.3", + "ws": "^8.14.2", + "youtube-transcript": "^1.0.6", + "youtubei.js": "^9.1.0" }, "peerDependenciesMeta": { - "@aws-sdk/client-dynamodb": { + "@aws-sdk/client-s3": { "optional": true }, - "@aws-sdk/client-lambda": { + "@aws-sdk/client-sagemaker-runtime": { "optional": true }, - "@aws-sdk/client-s3": { + "@aws-sdk/client-sfn": { "optional": true }, - "@clickhouse/client": { + "@aws-sdk/credential-provider-node": { "optional": true }, - "@getmetal/metal-sdk": { + "@azure/storage-blob": { "optional": true }, - "@huggingface/inference": { + "@browserbasehq/sdk": { "optional": true }, - "@opensearch-project/opensearch": { + "@gomomento/sdk": { + "optional": true + }, + "@gomomento/sdk-core": { + "optional": true + }, + "@gomomento/sdk-web": { + "optional": true + }, + "@mendable/firecrawl-js": { + "optional": true + }, + "@notionhq/client": { "optional": true }, "@pinecone-database/pinecone": { @@ -1474,16 +2551,16 @@ "@supabase/supabase-js": { "optional": true }, - "@tensorflow-models/universal-sentence-encoder": { + "@vercel/kv": { "optional": true }, - "@tensorflow/tfjs-converter": { + "@xata.io/client": { "optional": true }, - "@tensorflow/tfjs-core": { + "apify-client": { "optional": true }, - "@zilliz/milvus2-sdk-node": { + "assemblyai": { "optional": true }, "axios": { @@ -1495,7 +2572,10 @@ "chromadb": { "optional": true }, - "cohere-ai": { + "convex": { + "optional": true + }, + "couchbase": { "optional": true }, "d3-dsv": { @@ -1504,31 +2584,61 @@ "epub2": { "optional": true }, - "hnswlib-node": { + "faiss-node": { + "optional": true + }, + "fast-xml-parser": { + "optional": true + }, + "handlebars": { "optional": true }, "html-to-text": { "optional": true }, + "ignore": { + "optional": true + }, + "ioredis": { + "optional": true + }, + "jsdom": { + "optional": true + }, "mammoth": { "optional": true }, "mongodb": { "optional": true }, + "node-llama-cpp": { + "optional": true + }, + "notion-to-md": { + "optional": true + }, + "officeparser": { + "optional": true + }, "pdf-parse": { "optional": true }, + "peggy": { + "optional": true + }, "playwright": { "optional": true }, "puppeteer": { "optional": true }, + "pyodide": { + "optional": true + }, "redis": { "optional": true }, - "replicate": { + "sonix-speech-recognition": { "optional": true }, "srt-parser-2": { @@ -1539,6 +2649,51 @@ }, "weaviate-ts-client": { "optional": true + }, + "web-auth-library": { + "optional": true + }, + "ws": { + "optional": true + }, + "youtube-transcript": { + "optional": true + }, + "youtubei.js": { + "optional": true + } + } + }, + "node_modules/langchainhub": { + "version": "0.0.11", + "resolved": "https://registry.npmjs.org/langchainhub/-/langchainhub-0.0.11.tgz", + "integrity": "sha512-WnKI4g9kU2bHQP136orXr2bcRdgz9iiTBpTN0jWt9IlScUKnJBoD0aa2HOzHURQKeQDnt2JwqVmQ6Depf5uDLQ==" + }, + "node_modules/langsmith": { + "version": "0.1.32", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.1.32.tgz", + "integrity": "sha512-EUWHIH6fiOCGRYdzgwGoXwJxCMyUrL+bmUcxoVmkXoXoAGDOVinz8bqJLKbxotsQWqM64NKKsW85OTIutgNaMQ==", + "dependencies": { + "@types/uuid": "^9.0.1", + "commander": "^10.0.1", + "p-queue": "^6.6.2", + "p-retry": "4", + "uuid": "^9.0.0" + }, + "peerDependencies": { + "@langchain/core": "*", + "langchain": "*", + "openai": "*" + }, + "peerDependenciesMeta": { + "@langchain/core": { + "optional": true + }, + "langchain": { + "optional": true + }, + "openai": { + "optional": true } } }, @@ -1566,6 +2721,14 @@ "loose-envify": "cli.js" } }, + "node_modules/lru-cache": { + "version": "10.2.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", + "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", + "engines": { + "node": "14 || >=16.14" + } + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", @@ -1575,11 +2738,11 @@ } }, "node_modules/micromatch": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.5.tgz", - "integrity": "sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==", + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", + "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", "dependencies": { - "braces": "^3.0.2", + "braces": "^3.0.3", "picomatch": "^2.3.1" }, "engines": { @@ -1606,14 +2769,25 @@ } }, "node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "version": "9.0.4", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", + "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^2.0.1" }, "engines": { - "node": "*" + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/minipass": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", + "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==", + "engines": { + "node": ">=16 || 14 >=14.17" } }, "node_modules/ml-array-mean": { @@ -1633,9 +2807,9 @@ } }, "node_modules/ml-distance": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/ml-distance/-/ml-distance-4.0.0.tgz", - "integrity": "sha512-zj7+UGZpHk3uL7n79XTfGNUjIGnhLn8xVvrxYvBHvXFxo3jq1q+/UjP311hZxnLVhbxbXCjUniThX8gozjacYA==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/ml-distance/-/ml-distance-4.0.1.tgz", + "integrity": "sha512-feZ5ziXs01zhyFUUUeZV5hwc0f5JW0Sh0ckU1koZe/wdVkJdGxcP06KNQuF0WBTj8FttQUzcvQcpcrOp/XrlEw==", "dependencies": { "ml-array-mean": "^1.1.6", "ml-distance-euclidean": "^2.0.0", @@ -1661,6 +2835,14 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mz": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/mz/-/mz-2.7.0.tgz", @@ -1672,9 +2854,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.6", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", - "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "version": "3.3.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", + "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", "funding": [ { "type": "github", @@ -1741,11 +2923,6 @@ } } }, - "node_modules/next-tick": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/next-tick/-/next-tick-1.1.0.tgz", - "integrity": "sha512-CXdUiJembsNjuToQvxayPZF9Vqht7hewsvy2sOWafLvi2awflj9mOC6bHIg50orX8IJvWKY9wYQ/zB2kogPslQ==" - }, "node_modules/next/node_modules/postcss": { "version": "8.4.14", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.14.tgz", @@ -1769,10 +2946,31 @@ "node": "^10 || ^12 || >=14" } }, - "node_modules/node-addon-api": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-6.1.0.tgz", - "integrity": "sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA==" + "node_modules/next/node_modules/zod": { + "version": "3.21.4", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.21.4.tgz", + "integrity": "sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "engines": { + "node": ">=10.5.0" + } }, "node_modules/node-ensure": { "version": "0.0.0", @@ -1780,9 +2978,9 @@ "integrity": "sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==" }, "node_modules/node-fetch": { - "version": "2.6.7", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.7.tgz", - "integrity": "sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==", + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", "dependencies": { "whatwg-url": "^5.0.0" }, @@ -1798,20 +2996,10 @@ } } }, - "node_modules/node-gyp-build": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.6.0.tgz", - "integrity": "sha512-NTZVKn9IylLwUzaKjkas1e4u2DLNcV4rdYagA4PWdPwW87Bi7z+BznyKSRwS/761tV/lzCGXplWsiaMjLqP2zQ==", - "bin": { - "node-gyp-build": "bin.js", - "node-gyp-build-optional": "optional.js", - "node-gyp-build-test": "build-test.js" - } - }, "node_modules/node-releases": { - "version": "2.0.10", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.10.tgz", - "integrity": "sha512-5GFldHPXVG/YZmFzJvKK2zDSzPKhEp0+ZR5SVaoSag9fsL5YgHbUHDfnG5494ISANDcK4KwPXAx2xqVEydmd7w==" + "version": "2.0.14", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", + "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==" }, "node_modules/normalize-path": { "version": "3.0.0", @@ -1867,18 +3055,10 @@ "node": ">= 6" } }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dependencies": { - "wrappy": "1" - } - }, "node_modules/openai": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/openai/-/openai-3.2.1.tgz", - "integrity": "sha512-762C9BNlJPbjjlWZi4WYK9iM2tAVAv0uUp1UmI34vb0CN5T2mjB/qM6RYBmNKMh/dN9fC+bxqPwWJZUTWW052A==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-3.3.0.tgz", + "integrity": "sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==", "dependencies": { "axios": "^0.26.0", "form-data": "^4.0.0" @@ -1892,6 +3072,11 @@ "follow-redirects": "^1.14.8" } }, + "node_modules/openapi-types": { + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/openapi-types/-/openapi-types-12.1.3.tgz", + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==" + }, "node_modules/p-finally": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", @@ -1961,12 +3146,12 @@ "url": "https://github.com/inikulin/parse5?sponsor=1" } }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", + "node_modules/path-key": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", + "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", "engines": { - "node": ">=0.10.0" + "node": ">=8" } }, "node_modules/path-parse": { @@ -1974,6 +3159,21 @@ "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, + "node_modules/path-scurry": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.11.1.tgz", + "integrity": "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA==", + "dependencies": { + "lru-cache": "^10.2.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.18" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/pdf-parse": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/pdf-parse/-/pdf-parse-1.1.1.tgz", @@ -1994,21 +3194,10 @@ "ms": "^2.1.1" } }, - "node_modules/phin": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/phin/-/phin-3.7.0.tgz", - "integrity": "sha512-DqnVNrpYhKGBZppNKprD+UJylMeEKOZxHgPB+ZP6mGzf3uA2uox4Ep9tUm+rUc8WLIdHT3HcAE4X8fhwQA9JKg==", - "dependencies": { - "centra": "^2.6.0" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/picocolors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==" + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", + "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==" }, "node_modules/picomatch": { "version": "2.3.1", @@ -2030,9 +3219,9 @@ } }, "node_modules/pirates": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.5.tgz", - "integrity": "sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.6.tgz", + "integrity": "sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg==", "engines": { "node": ">= 6" } @@ -2099,20 +3288,26 @@ } }, "node_modules/postcss-load-config": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-4.0.1.tgz", - "integrity": "sha512-vEJIc8RdiBRu3oRAI0ymerOn+7rPuMvRXslTvZUKZonDHFIczxztIyJ1urxM1x9JXEikvpWWTUUqal5j/8QgvA==", + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/postcss-load-config/-/postcss-load-config-4.0.2.tgz", + "integrity": "sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ==", + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], "dependencies": { - "lilconfig": "^2.0.5", - "yaml": "^2.1.1" + "lilconfig": "^3.0.0", + "yaml": "^2.3.4" }, "engines": { "node": ">= 14" }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, "peerDependencies": { "postcss": ">=8.0.9", "ts-node": ">=9.0.0" @@ -2126,6 +3321,17 @@ } } }, + "node_modules/postcss-load-config/node_modules/lilconfig": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/lilconfig/-/lilconfig-3.1.2.tgz", + "integrity": "sha512-eop+wDAvpItUys0FWkHIKeC9ybYrTGbU41U5K7+bttZZeohvnY7M9dZ5kB21GNWiFT2q1OoPTvncPCgSOVO5ow==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/antonk52" + } + }, "node_modules/postcss-nested": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/postcss-nested/-/postcss-nested-6.0.1.tgz", @@ -2145,9 +3351,9 @@ } }, "node_modules/postcss-selector-parser": { - "version": "6.0.12", - "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.12.tgz", - "integrity": "sha512-NdxGCAZdRrwVI1sy59+Wzrh+pMMHxapGnpfenDVlMEXoOcvt4pGE0JLK9YY2F5dLxcFYA/YbVQKhcGU+FtSYQg==", + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.1.0.tgz", + "integrity": "sha512-UMz42UD0UY0EApS0ZL9o1XnLhSTtvvvLe5Dc2H2O56fvRZi+KulDyf5ctDhhtYJBGKStV2FL1fy6253cmLgqVQ==", "dependencies": { "cssesc": "^3.0.0", "util-deprecate": "^1.0.2" @@ -2176,6 +3382,14 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "node_modules/punycode": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", + "integrity": "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==", + "engines": { + "node": ">=6" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -2261,22 +3475,19 @@ "node": ">=8.10.0" } }, - "node_modules/resolve": { - "version": "1.22.2", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.2.tgz", - "integrity": "sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g==", - "dependencies": { - "is-core-module": "^2.11.0", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" + "node_modules/require-from-string": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "engines": { + "node": ">=0.10.0" } }, + "node_modules/resolve": { + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.1.7.tgz", + "integrity": "sha512-9znBF0vBcaSN3W2j7wKvdERPwqTxSpCq+if5C0WoTCyV9n24rua28jeuQ2pL/HOf+yUe/Mef+H/5p60K0Id3bg==" + }, "node_modules/retry": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", @@ -2316,10 +3527,15 @@ "queue-microtask": "^1.2.2" } }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, "node_modules/scheduler": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.0.tgz", - "integrity": "sha512-CtuThmgHNg7zIZWAXi3AsyIzA3n4xx7aNyjwC2VJldO2LMVDhFK+63xGqq6CsJH4rTAt6/M+N4GhZiDYPx9eUw==", + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", "dependencies": { "loose-envify": "^1.1.0" } @@ -2332,6 +3548,36 @@ "undici": "^5.12.0" } }, + "node_modules/shebang-command": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", + "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", + "dependencies": { + "shebang-regex": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/shebang-regex": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", + "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", + "engines": { + "node": ">=8" + } + }, + "node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/snapsvg": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/snapsvg/-/snapsvg-0.5.1.tgz", @@ -2364,9 +3610,9 @@ } }, "node_modules/source-map-js": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", - "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", + "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", "engines": { "node": ">=0.10.0" } @@ -2379,6 +3625,94 @@ "node": ">=10.0.0" } }, + "node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/string-width-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, "node_modules/styled-jsx": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/styled-jsx/-/styled-jsx-5.1.1.tgz", @@ -2402,13 +3736,13 @@ } }, "node_modules/sucrase": { - "version": "3.32.0", - "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.32.0.tgz", - "integrity": "sha512-ydQOU34rpSyj2TGyz4D2p8rbktIOZ8QY9s+DGLvFU1i5pWJE8vkpruCjGCMHsdXwnD7JDcS+noSwM/a7zyNFDQ==", + "version": "3.35.0", + "resolved": "https://registry.npmjs.org/sucrase/-/sucrase-3.35.0.tgz", + "integrity": "sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA==", "dependencies": { "@jridgewell/gen-mapping": "^0.3.2", "commander": "^4.0.0", - "glob": "7.1.6", + "glob": "^10.3.10", "lines-and-columns": "^1.1.6", "mz": "^2.7.0", "pirates": "^4.0.1", @@ -2419,13 +3753,21 @@ "sucrase-node": "bin/sucrase-node" }, "engines": { - "node": ">=8" + "node": ">=16 || 14 >=14.17" + } + }, + "node_modules/sucrase/node_modules/commander": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz", + "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==", + "engines": { + "node": ">= 6" } }, "node_modules/supports-color": { - "version": "9.3.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-9.3.1.tgz", - "integrity": "sha512-knBY82pjmnIzK3NifMo3RxEIRD9E0kIzV4BKcyTZ9+9kWgLMxd4PrsTSMoFQUabgRBbF8KOLRDCyKgNV+iK44Q==", + "version": "9.4.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-9.4.0.tgz", + "integrity": "sha512-VL+lNrEoIXww1coLPOmiEmK/0sGigko5COxI09KzHc2VJXJsQ37UaQ+8quuxjDeA7+KnLGTWRyOXSLLR2Wb4jw==", "engines": { "node": ">=12" }, @@ -2481,6 +3823,22 @@ "node": ">=14.0.0" } }, + "node_modules/tailwindcss/node_modules/resolve": { + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", + "dependencies": { + "is-core-module": "^2.13.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/thenify": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/thenify/-/thenify-3.3.1.tgz", @@ -2527,22 +3885,9 @@ "integrity": "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==" }, "node_modules/tslib": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", - "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" - }, - "node_modules/type": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/type/-/type-1.2.0.tgz", - "integrity": "sha512-+5nt5AAniqsCnu2cEQQdpzCAh33kVx8n0VoFidKpB1dVVLAN/F+bgVOqOJqOnEnrhp222clB5p3vUlD+1QAnfg==" - }, - "node_modules/typedarray-to-buffer": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", - "integrity": "sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==", - "dependencies": { - "is-typedarray": "^1.0.0" - } + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" }, "node_modules/typescript": { "version": "5.0.4", @@ -2558,20 +3903,25 @@ } }, "node_modules/undici": { - "version": "5.22.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.22.0.tgz", - "integrity": "sha512-fR9RXCc+6Dxav4P9VV/sp5w3eFiSdOjJYsbtWfd4s5L5C4ogyuVpdKIVHeW0vV1MloM65/f7W45nR9ZxwVdyiA==", + "version": "5.28.4", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.28.4.tgz", + "integrity": "sha512-72RFADWFqKmUb2hmmvNODKL3p9hcB6Gt2DOQMis1SEBaV6a4MH8soBvzg+95CYhCKPFedut2JY9bMfrDl9D23g==", "dependencies": { - "busboy": "^1.6.0" + "@fastify/busboy": "^2.0.0" }, "engines": { "node": ">=14.0" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/update-browserslist-db": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.11.tgz", - "integrity": "sha512-dCwEFf0/oT85M1fHBg4F0jtLwJrutGoHSQXCh7u4o2t1drG+c0a9Flnqww6XUKSfQMPpJBRjU8d4RXB09qtvaA==", + "version": "1.0.16", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.16.tgz", + "integrity": "sha512-KVbTxlBYlckhF5wgfyZXTWnMn7MMZjMu9XG8bPlliUOP9ThaF4QnhP8qrjrH7DRzHfSk0oQv1wToW+iA5GajEQ==", "funding": [ { "type": "opencollective", @@ -2587,8 +3937,8 @@ } ], "dependencies": { - "escalade": "^3.1.1", - "picocolors": "^1.0.0" + "escalade": "^3.1.2", + "picocolors": "^1.0.1" }, "bin": { "update-browserslist-db": "cli.js" @@ -2597,16 +3947,12 @@ "browserslist": ">= 4.21.0" } }, - "node_modules/utf-8-validate": { - "version": "5.0.10", - "resolved": "https://registry.npmjs.org/utf-8-validate/-/utf-8-validate-5.0.10.tgz", - "integrity": "sha512-Z6czzLq4u8fPOyx7TU6X3dvUZVvoJmxSQ+IcrlmagKhilxlhZgxPK6C5Jqbkw1IDUmFTM+cz9QDnnLTwDz/2gQ==", - "hasInstallScript": true, + "node_modules/uri-js": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", "dependencies": { - "node-gyp-build": "^4.3.0" - }, - "engines": { - "node": ">=6.14.2" + "punycode": "^2.1.0" } }, "node_modules/util-deprecate": { @@ -2615,9 +3961,13 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, "node_modules/uuid": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", - "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], "bin": { "uuid": "dist/bin/uuid" } @@ -2630,92 +3980,190 @@ "node": ">= 0.8" } }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "engines": { + "node": ">= 8" + } + }, "node_modules/webidl-conversions": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, - "node_modules/websocket": { - "version": "1.0.34", - "resolved": "https://registry.npmjs.org/websocket/-/websocket-1.0.34.tgz", - "integrity": "sha512-PRDso2sGwF6kM75QykIesBijKSVceR6jL2G8NGYyq2XrItNC2P5/qL5XeR056GhA+Ly7JMFvJb9I312mJfmqnQ==", + "node_modules/whatwg-url": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", "dependencies": { - "bufferutil": "^4.0.1", - "debug": "^2.2.0", - "es5-ext": "^0.10.50", - "typedarray-to-buffer": "^3.1.5", - "utf-8-validate": "^5.0.2", - "yaeti": "^0.0.6" + "tr46": "~0.0.3", + "webidl-conversions": "^3.0.0" + } + }, + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" }, "engines": { - "node": ">=4.0.0" + "node": ">= 8" } }, - "node_modules/websocket/node_modules/debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", "dependencies": { - "ms": "2.0.0" + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/websocket/node_modules/ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "node_modules/wrap-ansi-cjs/node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" + "node_modules/wrap-ansi-cjs/node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, - "node_modules/yaeti": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/yaeti/-/yaeti-0.0.6.tgz", - "integrity": "sha512-MvQa//+KcZCUkBTIC9blM+CU9J2GzuTytsOUwf2lidtvkx/6gnEp1QvJv34t9vdjhFmha/mUiNDbN0D0mJWdug==", + "node_modules/wrap-ansi-cjs/node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/ws": { + "version": "8.17.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz", + "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==", "engines": { - "node": ">=0.10.32" + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } } }, "node_modules/yaml": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.2.tgz", - "integrity": "sha512-CBKFWExMn46Foo4cldiChEzn7S7SRV+wqiluAb6xmueD/fGyRHIhX8m14vVGgeFWjN540nKCNVj6P21eQjgTuA==", + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.4.5.tgz", + "integrity": "sha512-aBx2bnqDzVOyNKfsysjA2ms5ZlnjSAW2eG3/L5G/CSujfjLJTJsEw1bGw8kCf04KodQWk1pxlGnZ56CRxiawmg==", + "bin": { + "yaml": "bin.mjs" + }, "engines": { "node": ">= 14" } }, "node_modules/youtube-transcript": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/youtube-transcript/-/youtube-transcript-1.0.6.tgz", - "integrity": "sha512-k/6uxB9voj/5astl6+q+VArX/aWHhnmle8BucvUCTYTQQEOSVlBiXkrI0KD3o8A0b44MV6q0bmVNiJFIpTlcZA==", - "dependencies": { - "phin": "^3.5.0" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/youtube-transcript/-/youtube-transcript-1.2.1.tgz", + "integrity": "sha512-TvEGkBaajKw+B6y91ziLuBLsa5cawgowou+Bk0ciGpjELDfAzSzTGXaZmeSSkUeknCPpEr/WGApOHDwV7V+Y9Q==", + "engines": { + "node": ">=18.0.0" } }, "node_modules/zod": { - "version": "3.21.4", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.21.4.tgz", - "integrity": "sha512-m46AKbrzKVzOzs/DZgVnG5H55N1sv1M8qZU3A8RIKbs3mrACDNeIOeilDymVb2HdmP8uwshOCF4uJ8uM9rCqJw==", + "version": "3.23.8", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.23.8.tgz", + "integrity": "sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==", "funding": { "url": "https://github.com/sponsors/colinhacks" } }, "node_modules/zod-to-json-schema": { - "version": "3.21.0", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.21.0.tgz", - "integrity": "sha512-+KyFCzqKwE6CxMSZxEUBaGmdXzB09BoFebO+xef/ISE4cTfReQlyThYbS8aqd3uWkdt9fz5BGHsY0CbY+Ra9oA==", + "version": "3.23.1", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.23.1.tgz", + "integrity": "sha512-oT9INvydob1XV0v1d2IadrR74rLtDInLvDFfAa1CG0Pmg/vxATk7I2gSelfj271mbzeM4Da0uuDQE/Nkj3DWNw==", "peerDependencies": { - "zod": "^3.21.4" + "zod": "^3.23.3" } } } diff --git a/package.json b/package.json index 4ec22c7..22e5aae 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,11 @@ "lint": "next lint" }, "dependencies": { - "@pinecone-database/pinecone": "^0.1.5", + "@langchain/community": "^0.2.11", + "@langchain/openai": "^0.1.3", + "@langchain/pinecone": "^0.0.7", + "@langchain/textsplitters": "^0.0.3", + "@pinecone-database/pinecone": "^2.2.2", "@supabase/supabase-js": "^2.24.0", "autoprefixer": "10.4.14", "axios": "^1.4.0", @@ -17,8 +21,7 @@ "cors": "^2.8.5", "debug": "^4.3.4", "dotenv": "^16.0.3", - "hnswlib-node": "^1.4.2", - "langchain": "^0.0.75", + "langchain": "^0.2.5", "next": "13.4.1", "openai": "^3.2.1", "pdf-parse": "^1.1.1", @@ -43,5 +46,8 @@ "edge 105", "ios_saf 15.6", "firefox 104" - ] + ], + "overrides": { + "@langchain/core": "0.2.7" + } } From 2fe5b05db3bb06cc43748ae746dc98888ca23955 Mon Sep 17 00:00:00 2001 From: Shawn Esquivel <shawnesquivel24@gmail.com> Date: Tue, 18 Jun 2024 16:39:55 -0700 Subject: [PATCH 17/17] MAJOR UPDATE: Refactored for 2024 --- pages/api/pdf-query-soln.js | 65 +++++++++++++++ pages/api/pdf-query.js | 84 +++++-------------- pages/api/pdf-upload-soln.js | 63 +++++++++++++++ pages/api/pdf-upload.js | 153 ++++------------------------------- 4 files changed, 164 insertions(+), 201 deletions(-) create mode 100644 pages/api/pdf-query-soln.js create mode 100644 pages/api/pdf-upload-soln.js diff --git a/pages/api/pdf-query-soln.js b/pages/api/pdf-query-soln.js new file mode 100644 index 0000000..1c02c75 --- /dev/null +++ b/pages/api/pdf-query-soln.js @@ -0,0 +1,65 @@ +import { PDFLoader } from "@langchain/community/document_loaders/fs/pdf"; +import { Pinecone } from "@pinecone-database/pinecone"; +import { Document } from "@langchain/core/documents"; +import { OpenAIEmbeddings } from "@langchain/openai"; +import { PineconeStore } from "@langchain/pinecone"; +import { CharacterTextSplitter } from "@langchain/textsplitters"; + + +export default async function handler(req, res) { + // Validators + if (!process.env.PINECONE_ENVIRONMENT || !process.env.PINECONE_API_KEY) { + throw new Error("Pinecone environment or api key vars missing"); + } + if (req.method !== "GET") { + return res.status(405).json({ message: "Method not allowed" }); + } + + /** STEP ONE: LOAD DOCUMENT */ + const bookPath = + "/Users/shawnesquivel/GitHub/langchain-js-course/data/document_loaders/naval-ravikant-book.pdf"; + const loader = new PDFLoader(bookPath); + const docs = await loader.load(); + + + if (docs.length === 0) { + console.log("No documents found."); + throw new Error("No documents created from the resource."); + } + + const splitter = new CharacterTextSplitter({ + separator: " ", + chunkSize: 250, + chunkOverlap: 10, + }); + + const splitDocs = await splitter.splitDocuments(docs); + + // Reduce the size of the metadata for each document -- lots of useless pdf information + const reducedDocs = splitDocs.map((doc) => { + const reducedMetadata = { ...doc.metadata }; + delete reducedMetadata.pdf; // Remove the 'pdf' field + return new Document({ + pageContent: doc.pageContent, + metadata: reducedMetadata, + }); + }); + + // reference; https://docs.pinecone.io/reference/pinecone-clients#node-js-client + const client = new Pinecone({ apiKey: process.env.PINECONE_API_KEY }); + const pineconeIndex = client.Index(process.env.PINECONE_INDEX); + + try { + await PineconeStore.fromDocuments(reducedDocs, new OpenAIEmbeddings(), { + pineconeIndex, + }); + } catch (error) { + console.log('Error uploading to Pinecone:', error); + } + + + + return res.status(200).json({ + result: `Uploaded to Pinecone! Before splitting: ${docs.length}, After splitting: ${splitDocs.length}`, + }); +} diff --git a/pages/api/pdf-query.js b/pages/api/pdf-query.js index 8b50d57..c340b3a 100644 --- a/pages/api/pdf-query.js +++ b/pages/api/pdf-query.js @@ -1,72 +1,28 @@ -import { PineconeClient } from "@pinecone-database/pinecone"; -import { VectorDBQAChain } from "langchain/chains"; -import { OpenAIEmbeddings } from "langchain/embeddings/openai"; -import { OpenAI } from "langchain/llms/openai"; -import { PineconeStore } from "langchain/vectorstores/pinecone"; -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ +import { PDFLoader } from "@langchain/community/document_loaders/fs/pdf"; +import { Pinecone } from "@pinecone-database/pinecone"; +import { Document } from "@langchain/core/documents"; +import { OpenAIEmbeddings } from "@langchain/openai"; +import { PineconeStore } from "@langchain/pinecone"; +import { CharacterTextSplitter } from "@langchain/textsplitters"; -// Example: https://js.langchain.com/docs/modules/indexes/document_loaders/examples/file_loaders/pdf -export default async function handler(req, res) { - try { - if (req.method !== "POST") { - throw new Error("Method not allowed"); - } - - console.log("Query PDF"); - - // Grab the user prompt - const { input } = req.body; - - if (!input) { - throw new Error("No input"); - } - - console.log("input received:", input); - - // const client = new PineconeClient(); - // await client.init({ - // apiKey: process.env.PINECONE_API_KEY, - // environment: process.env.PINECONE_ENVIRONMENT, - // }); - // const pineconeIndex = client.Index(process.env.PINECONE_INDEX); - - const vectorStore = await PineconeStore.fromExistingIndex( - new OpenAIEmbeddings(), - { pineconeIndex } - ); - const privateKey = process.env.SUPABASE_PRIVATE_KEY; - if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); - - const url = process.env.SUPABASE_URL; - if (!url) throw new Error(`Expected env var SUPABASE_URL`); - - const client = createClient(url, privateKey); +export default async function handler(req, res) { + // Validators + if (!process.env.PINECONE_ENVIRONMENT || !process.env.PINECONE_API_KEY) { + throw new Error("Pinecone environment or api key vars missing"); + } + if (req.method !== "GET") { + return res.status(405).json({ message: "Method not allowed" }); + } - // Alternative: use Supabase instead of Pinecone if you're on waitlist (see this video for explanation: https://www.udemy.com/course/langchain-develop-ai-web-apps-with-javascript-and-langchain/learn/lecture/38362160) - // const vectorStore = await SupabaseVectorStore.fromExistingIndex( - // new OpenAIEmbeddings(), - // { client, tableName: "documents", queryName: "match_documents" } - // ); + /** Step 1: Create Documents */ - /* Part Two: Use as part of a chain (currently no metadata filters) */ - const model = new OpenAI(); - const chain = VectorDBQAChain.fromLLM(model, vectorStore, { - k: 1, - returnSourceDocuments: true, - }); - const response = await chain.call({ query: input }); + /** Step 2: Upload to Pinecone */ + // reference; https://docs.pinecone.io/reference/pinecone-clients#node-js-client - console.log(response); - return res.status(200).json({ result: response }); - } catch (error) { - console.error(error); - res.status(500).json({ message: error.message }); - } + return res.status(200).json({ + result: `Uploaded to Pinecone! `, + }); } diff --git a/pages/api/pdf-upload-soln.js b/pages/api/pdf-upload-soln.js new file mode 100644 index 0000000..9adde6d --- /dev/null +++ b/pages/api/pdf-upload-soln.js @@ -0,0 +1,63 @@ +import { PDFLoader } from "@langchain/community/document_loaders/fs/pdf"; +import { Pinecone } from "@pinecone-database/pinecone"; +import { Document } from "@langchain/core/documents"; +import { OpenAIEmbeddings } from "@langchain/openai"; +import { PineconeStore } from "@langchain/pinecone"; +import { CharacterTextSplitter } from "@langchain/textsplitters"; + + +export default async function handler(req, res) { + // Validators + if (!process.env.PINECONE_ENVIRONMENT || !process.env.PINECONE_API_KEY) { + throw new Error("Pinecone environment or api key vars missing"); + } + if (req.method !== "GET") { + return res.status(405).json({ message: "Method not allowed" }); + } + + /** STEP ONE: LOAD DOCUMENT */ + const bookPath = + "/Users/shawnesquivel/GitHub/langchain-js-course/data/document_loaders/naval-ravikant-book.pdf"; + const loader = new PDFLoader(bookPath); + const docs = await loader.load(); + + + if (docs.length === 0) { + console.log("No documents found."); + throw new Error("No documents created from the resource."); + } + + const splitter = new CharacterTextSplitter({ + separator: " ", + chunkSize: 250, + chunkOverlap: 10, + }); + + const splitDocs = await splitter.splitDocuments(docs); + + // Reduce the size of the metadata for each document -- lots of useless pdf information + const reducedDocs = splitDocs.map((doc) => { + const reducedMetadata = { ...doc.metadata }; + delete reducedMetadata.pdf; // Remove the 'pdf' field + return new Document({ + pageContent: doc.pageContent, + metadata: reducedMetadata, + }); + }); + + // reference; https://docs.pinecone.io/reference/pinecone-clients#node-js-client + const client = new Pinecone({ apiKey: process.env.PINECONE_API_KEY }); + const pineconeIndex = client.Index(process.env.PINECONE_INDEX); + + try { + await PineconeStore.fromDocuments(reducedDocs, new OpenAIEmbeddings(), { + pineconeIndex, + }); + } catch (error) { + console.log('Error uploading to Pinecone:', error); + } + + return res.status(200).json({ + result: `Uploaded to Pinecone! Before splitting: ${docs.length}, After splitting: ${splitDocs.length}`, + }); +} diff --git a/pages/api/pdf-upload.js b/pages/api/pdf-upload.js index fbd5532..6b68c21 100644 --- a/pages/api/pdf-upload.js +++ b/pages/api/pdf-upload.js @@ -1,143 +1,22 @@ -import { PDFLoader } from "langchain/document_loaders/fs/pdf"; -import { PineconeClient } from "@pinecone-database/pinecone"; -import { Document } from "langchain/document"; -import { OpenAIEmbeddings } from "langchain/embeddings/openai"; -import { PineconeStore } from "langchain/vectorstores/pinecone"; -import { CharacterTextSplitter } from "langchain/text_splitter"; +import { PDFLoader } from "@langchain/community/document_loaders/fs/pdf"; +import { Pinecone } from "@pinecone-database/pinecone"; +import { Document } from "@langchain/core/documents"; +import { OpenAIEmbeddings } from "@langchain/openai"; +import { PineconeStore } from "@langchain/pinecone"; +import { CharacterTextSplitter } from "@langchain/textsplitters"; -/** - * - * WARNING: THIS IS THE SOLUTION! Please try coding before viewing this. - * - */ - -// Example: https://js.langchain.com/docs/modules/indexes/document_loaders/examples/file_loaders/pdf - -/** - * - * INSTRUCTIONS - * 1. Run with book - error { - name: 'PineconeError', - source: 'server', - message: 'PineconeClient: Error calling upsert: PineconeError: metadata size is 140052 bytes, which exceeds the limit of 40960 bytes per vector', - stack: '' - } - * 2. Explain why -- vector meta data sizes is too big. - Language Models are often limited by the amount of text that you can pass to them. Therefore, it is neccessary to split them up into smaller chunks. LangChain provides several utilities for doing so. - https://js.langchain.com/docs/modules/indexes/text_splitters/ - - Play with chunk sizes... too small and you can't understand. - Fine tune this to your liking. - More vectors = more $$ - - - 3. Pinecone size 1536 - https://platform.openai.com/docs/guides/embeddings/second-generation-models - - 4. Upsert metadata size -- add this after split Docs - - // Reduce the size of the metadata for each document - const reducedDocs = splitDocs.map(doc => { - const reducedMetadata = { ...doc.metadata }; - delete reducedMetadata.pdf; // Remove the 'pdf' field - return new Document({ - pageContent: doc.pageContent, - metadata: reducedMetadata, - }); -}); - - - - - - * */ export default async function handler(req, res) { - if (req.method === "GET") { - console.log("Uploading book"); - // Enter your code here - /** STEP ONE: LOAD DOCUMENT */ - const bookPath = - "/Users/shawnesquivel/GitHub/yt-script-generator/data/document_loaders/naval-ravikant-book.pdf"; - const loader = new PDFLoader(bookPath); - - const docs = await loader.load(); - - if (docs.length === 0) { - console.log("No documents found."); - return; - } - - const splitter = new CharacterTextSplitter({ - separator: " ", - chunkSize: 250, - chunkOverlap: 10, - }); - - const splitDocs = await splitter.splitDocuments(docs); - - // Reduce the size of the metadata for each document -- lots of useless pdf information - const reducedDocs = splitDocs.map((doc) => { - const reducedMetadata = { ...doc.metadata }; - delete reducedMetadata.pdf; // Remove the 'pdf' field - return new Document({ - pageContent: doc.pageContent, - metadata: reducedMetadata, - }); - }); - - // docs.forEach((doc) => { - // console.log(doc); - // }); - - // console.log(`Uploading documents to Pinecone: ${docs}`); - - console.log(docs[100]); - console.log(splitDocs[100].metadata); - console.log(reducedDocs[100].metadata); - - /** STEP TWO: UPLOAD TO DATABASE */ - - const client = new PineconeClient(); - - await client.init({ - apiKey: process.env.PINECONE_API_KEY, - environment: process.env.PINECONE_ENVIRONMENT, - }); - - const pineconeIndex = client.Index(process.env.PINECONE_INDEX); - - await PineconeStore.fromDocuments(reducedDocs, new OpenAIEmbeddings(), { - pineconeIndex, - }); - - // Alternative: use Supabase instead of Pinecone if you're on waitlist (see this video for explanation: https://www.udemy.com/course/langchain-develop-ai-web-apps-with-javascript-and-langchain/learn/lecture/38362160) - // const privateKey = process.env.SUPABASE_PRIVATE_KEY; - // if (!privateKey) throw new Error(`Expected env var SUPABASE_PRIVATE_KEY`); - - // const url = process.env.SUPABASE_URL; - // if (!url) throw new Error(`Expected env var SUPABASE_URL`); - - // const client = createClient(url, privateKey); + // Validators + if (!process.env.PINECONE_ENVIRONMENT || !process.env.PINECONE_API_KEY) { + throw new Error("Pinecone environment or api key vars missing"); + } + if (req.method !== "GET") { + return res.status(405).json({ message: "Method not allowed" }); + } - /** UPLOAD TO SUPABASE */ - await SupabaseVectorStore.fromDocuments( - reducedDocs, - new OpenAIEmbeddings(), - { - client, - tableName: "documents", - queryName: "match_documents", - } - ); - console.log("Successfully uploaded to DB"); - // Modify output as needed - return res.status(200).json({ - result: `Uploaded to Pinecone! Before splitting: ${docs.length}, After splitting: ${splitDocs.length}`, - }); - } else { - res.status(405).json({ message: "Method not allowed" }); - } + return res.status(200).json({ + result: `Uploaded to Pinecone! `, + }); }