diff --git a/.gitignore b/.gitignore index 96f044b59..67374d2fe 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ TODO.md keys ERROR.png flowchart-fun.feature-reacher.json -.parcel-cache \ No newline at end of file +.parcel-cache + +speech*.mp4 \ No newline at end of file diff --git a/api/_lib/_llm.ts b/api/_lib/_llm.ts new file mode 100644 index 000000000..9db8e18b4 --- /dev/null +++ b/api/_lib/_llm.ts @@ -0,0 +1,70 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { z, ZodObject } from "zod"; +import { openai } from "./_openai"; +import zodToJsonSchema from "zod-to-json-schema"; +import OpenAI from "openai"; + +type Schemas>> = T; + +export async function llmMany>>( + content: string, + schemas: Schemas +) { + try { + // if the user passes a key "message" in schemas, throw an error + if (schemas.message) throw new Error("Cannot use key 'message' in schemas"); + + const completion = await openai.chat.completions.create({ + messages: [ + { + role: "user", + content, + }, + ], + tools: Object.entries(schemas).map(([key, schema]) => ({ + type: "function", + function: { + name: key, + parameters: zodToJsonSchema(schema), + }, + })), + model: "gpt-3.5-turbo-1106", + // model: "gpt-4-1106-preview", + }); + + const choice = completion.choices[0]; + + if (!choice) throw new Error("No choices returned"); + + // Must return the full thing, message and multiple tool calls + return simplifyChoice(choice) as SimplifiedChoice; + } catch (error) { + console.error(error); + const message = (error as Error)?.message || "Error with prompt"; + throw new Error(message); + } +} + +type SimplifiedChoice>> = { + message: string; + toolCalls: Array< + { + [K in keyof T]: { + name: K; + args: z.infer; + }; + }[keyof T] + >; +}; + +function simplifyChoice(choice: OpenAI.Chat.Completions.ChatCompletion.Choice) { + return { + message: choice.message.content || "", + toolCalls: + choice.message.tool_calls?.map((toolCall) => ({ + name: toolCall.function.name, + // Wish this were type-safe! + args: JSON.parse(toolCall.function.arguments ?? "{}"), + })) || [], + }; +} diff --git a/api/package.json b/api/package.json index 2c5a777aa..cd021e8ad 100644 --- a/api/package.json +++ b/api/package.json @@ -16,21 +16,27 @@ "@sendgrid/mail": "^7.4.6", "@supabase/supabase-js": "^2.31.0", "ajv": "^8.12.0", + "axios": "^0.27.2", "csv-parse": "^5.3.6", "date-fns": "^2.29.3", - "graph-selector": "^0.9.11", + "graph-selector": "^0.10.0", "highlight.js": "^11.8.0", "marked": "^4.1.1", + "micro": "^10.0.1", "moniker": "^0.1.2", + "multer": "1.4.5-lts.1", "notion-to-md": "^2.5.5", - "openai": "^4.10.0", + "openai": "^4.24.2", "shared": "workspace:*", - "stripe": "^11.11.0" + "stripe": "^11.11.0", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" }, "devDependencies": { "@swc/jest": "^0.2.24", "@types/jest": "^29.0.0", "@types/marked": "^4.0.7", + "@types/multer": "^1.4.11", "@types/node": "^18.16.17", "@typescript-eslint/eslint-plugin": "^6.2.0", "@typescript-eslint/parser": "^6.2.0", diff --git a/api/prompt/edit.ts b/api/prompt/edit.ts new file mode 100644 index 000000000..60a7898eb --- /dev/null +++ b/api/prompt/edit.ts @@ -0,0 +1,42 @@ +import { VercelApiHandler } from "@vercel/node"; +import { llmMany } from "../_lib/_llm"; +import { z } from "zod"; + +const nodeSchema = z.object({ + // id: z.string(), + // classes: z.string(), + label: z.string(), +}); + +const edgeSchema = z.object({ + from: z.string(), + to: z.string(), + label: z.string().optional().default(""), +}); + +const graphSchema = z.object({ + nodes: z.array(nodeSchema), + edges: z.array(edgeSchema), +}); + +const handler: VercelApiHandler = async (req, res) => { + const { graph, prompt } = req.body; + if (!graph || !prompt) { + throw new Error("Missing graph or prompt"); + } + + const result = await llmMany( + `You are a one-shot AI flowchart assistant. Help the user with a flowchart or diagram. Here is the current state of the flowchart: +${JSON.stringify(graph, null, 2)} + +Here is the user's message: +${prompt}`, + { + updateGraph: graphSchema, + } + ); + + res.json(result); +}; + +export default handler; diff --git a/api/prompt/speech-to-text.ts b/api/prompt/speech-to-text.ts new file mode 100644 index 000000000..d85e60892 --- /dev/null +++ b/api/prompt/speech-to-text.ts @@ -0,0 +1,27 @@ +import { VercelApiHandler } from "@vercel/node"; +import { openai } from "../_lib/_openai"; +import { toFile } from "openai"; + +const handler: VercelApiHandler = async (req, res) => { + try { + const { audioUrl } = req.body; + + if (!audioUrl) { + res.status(400).json({ ok: false, error: "No audioUrl provided" }); + return; + } + + const base64Data = audioUrl.split(";base64,").pop(); + const binaryData = Buffer.from(base64Data, "base64"); + const transcription = await openai.audio.transcriptions.create({ + file: await toFile(binaryData, "audio.mp4"), + model: "whisper-1", + }); + res.send(transcription.text); + } catch (error) { + console.error(error); + res.status(500).json({ ok: false, error: "Something went wrong" }); + } +}; + +export default handler; diff --git a/app/package.json b/app/package.json index 16b6c2f27..445d813a9 100644 --- a/app/package.json +++ b/app/package.json @@ -46,6 +46,7 @@ "@radix-ui/react-select": "^1.2.0", "@radix-ui/react-slider": "^1.1.2", "@radix-ui/react-tabs": "^1.0.4", + "@radix-ui/react-toast": "^1.1.5", "@radix-ui/react-toggle": "^1.0.3", "@radix-ui/react-tooltip": "^1.0.7", "@react-hook/throttle": "^2.2.0", @@ -77,7 +78,7 @@ "file-saver": "^2.0.5", "formulaic": "workspace:*", "framer-motion": "^10.13.1", - "graph-selector": "^0.9.12", + "graph-selector": "^0.10.0", "gray-matter": "^4.0.2", "highlight.js": "^11.7.0", "immer": "^9.0.16", diff --git a/app/public/images/ai-edit.png b/app/public/images/ai-edit.png new file mode 100644 index 000000000..4aa5853bd Binary files /dev/null and b/app/public/images/ai-edit.png differ diff --git a/app/scripts/autotranslations.mjs b/app/scripts/autotranslations.mjs index d2b8163f9..e85080e89 100644 --- a/app/scripts/autotranslations.mjs +++ b/app/scripts/autotranslations.mjs @@ -139,7 +139,7 @@ for (const locale of locales) { `Translating ${batch.length} phrases... (${retries} retries)` ); const response = await openai.createCompletion({ - model: "text-davinci-003", + model: "gpt-3.5-turbo-instruct", prompt, max_tokens: 2048, temperature: 0.5, diff --git a/app/src/components/App.tsx b/app/src/components/App.tsx index c589850a6..b66cefd55 100644 --- a/app/src/components/App.tsx +++ b/app/src/components/App.tsx @@ -19,6 +19,7 @@ import { Trans } from "@lingui/macro"; import { House, Recycle } from "phosphor-react"; import { ReactQueryDevtools } from "react-query/devtools"; import { BrowserRouter } from "react-router-dom"; +import * as Toast from "@radix-ui/react-toast"; import { Button2 } from "../ui/Shared"; import Loading from "./Loading"; @@ -35,9 +36,12 @@ export default function App() { }> - - - + + + + + + diff --git a/app/src/components/EditWithAI.tsx b/app/src/components/EditWithAI.tsx new file mode 100644 index 000000000..23ca9365c --- /dev/null +++ b/app/src/components/EditWithAI.tsx @@ -0,0 +1,281 @@ +import { MagicWand, Robot } from "phosphor-react"; +import { Button2 } from "../ui/Shared"; +import * as Popover from "@radix-ui/react-popover"; +import { Trans, t } from "@lingui/macro"; +import { useCallback, useRef, useState } from "react"; +import { useDoc } from "../lib/useDoc"; +import { parse, stringify, Graph as GSGraph } from "graph-selector"; +import { useMutation } from "react-query"; +import * as Toast from "@radix-ui/react-toast"; +import { Microphone } from "./Microphone"; +import { useIsProUser } from "../lib/hooks"; +import { showPaywall } from "../lib/usePaywallModalStore"; + +// The Graph type we send to AI is slightly different from internal representation +type GraphForAI = { + nodes: { + label: string; + id?: string; + }[]; + edges: { + label: string; + from: string; + to: string; + }[]; +}; + +const title = t`AI-Powered Diagramming`; +const content = t`With Flowchart Fun's Pro version, you can tap into AI to quickly flesh out your flowchart details, ideal for creating diagrams on the go. For $3/month, get the ease of accessible AI editing to enhance your flowcharting experience.`; + +export function EditWithAI() { + const [message, setMessage] = useState(null); + const [isOpen, setIsOpen] = useState(false); + const [transcriptionLoading, setTranscriptionLoading] = useState(false); + const isProUser = useIsProUser(); + + const { mutate: edit, isLoading: editIsLoading } = useMutation({ + mutationFn: async (body: { prompt: string; graph: GraphForAI }) => { + // /api/prompt/edit + const response = await fetch("/api/prompt/edit", { + method: "POST", + body: JSON.stringify(body), + headers: { + "Content-Type": "application/json", + }, + }); + const data = await response.json(); + + return data as { + message: string; + toolCalls: { + name: "updateGraph"; + args: GraphForAI; + }[]; + }; + }, + onSuccess(data) { + if (data.message) { + setMessage(data.message); + } + + for (const { name, args } of data.toolCalls) { + switch (name) { + case "updateGraph": { + const newText = toGraphSelector(args); + useDoc.setState({ text: newText }, false, "EditWithAI"); + break; + } + } + } + }, + onSettled() { + setTranscriptionLoading(false); + }, + }); + + const submitPrompt = useCallback( + (prompt: string) => { + if (!isProUser) { + showPaywall({ + title, + content, + imgUrl: "/images/ai-edit.png", + }); + return; + } + + setIsOpen(false); + + const text = useDoc.getState().text; + const _graph = parse(text); + + const graph: GraphForAI = { + nodes: _graph.nodes.map((node) => { + if (isCustomID(node.data.id)) { + return { + label: node.data.label, + id: node.data.id, + }; + } + + return { + label: node.data.label, + }; + }), + edges: _graph.edges.map((edge) => { + // Because generated edges internally use a custom ID, + // we need to find the label, unless the user is using a custom ID + + let from = edge.source; + if (!isCustomID(from)) { + // find the from node + const fromNode = _graph.nodes.find((node) => node.data.id === from); + if (!fromNode) throw new Error("from node not found"); + from = fromNode.data.label; + } + + let to = edge.target; + if (!isCustomID(to)) { + // find the to node + const toNode = _graph.nodes.find((node) => node.data.id === to); + if (!toNode) throw new Error("to node not found"); + to = toNode.data.label; + } + + return { + label: edge.data.label, + from, + to, + }; + }), + }; + + edit({ prompt, graph }); + }, + [edit, isProUser] + ); + + const handleSubmit = useCallback( + (e: React.FormEvent) => { + e.preventDefault(); + + const formData = new FormData(e.currentTarget); + const prompt = formData.get("prompt") as string; + if (!prompt) return; + + submitPrompt(prompt); + }, + [submitPrompt] + ); + + const handleSend = useCallback(() => { + if (isProUser) { + setTranscriptionLoading(true); + setIsOpen(false); + } else { + showPaywall({ + title, + content, + imgUrl: "/images/ai-edit.png", + }); + return; + } + }, [isProUser]); + + const formRef = useRef(null); + + const isLoading = editIsLoading || transcriptionLoading; + + return ( + <> + + + + } + color="purple" + size="sm" + rounded + className="aria-[expanded=true]:bg-purple-700 !pt-2 !pb-[9px] !pl-3 !pr-4" + isLoading={isLoading} + > + + Edit with AI + + + + + +
+
+