-
-
Notifications
You must be signed in to change notification settings - Fork 232
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
c9bccf4
commit 0bb50f5
Showing
7 changed files
with
320 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,4 +12,6 @@ TODO.md | |
| keys | ||
| ERROR.png | ||
| flowchart-fun.feature-reacher.json | ||
| .parcel-cache | ||
| .parcel-cache | ||
|
|
||
| speech*.mp4 | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| import { VercelApiHandler } from "@vercel/node"; | ||
| import { openai } from "../_lib/_openai"; | ||
| import { toFile } from "openai"; | ||
|
|
||
| const handler: VercelApiHandler = async (req, res) => { | ||
| try { | ||
| const { audioUrl } = req.body; | ||
|
|
||
| if (!audioUrl) { | ||
| res.status(400).json({ ok: false, error: "No audioUrl provided" }); | ||
| return; | ||
| } | ||
|
|
||
| const base64Data = audioUrl.split(";base64,").pop(); | ||
| const binaryData = Buffer.from(base64Data, "base64"); | ||
| const transcription = await openai.audio.transcriptions.create({ | ||
| file: await toFile(binaryData, "audio.mp4"), | ||
| model: "whisper-1", | ||
| }); | ||
| res.send(transcription.text); | ||
| } catch (error) { | ||
| console.error(error); | ||
| res.status(500).json({ ok: false, error: "Something went wrong" }); | ||
| } | ||
| }; | ||
|
|
||
| export default handler; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| import { Microphone as MicrophoneIcon } from "phosphor-react"; | ||
| import { IconButton2 } from "../ui/Shared"; | ||
| import { useCallback } from "react"; | ||
| import cx from "classnames"; | ||
| import { | ||
| startRecording, | ||
| stopRecording, | ||
| turnOnMicrophone, | ||
| useMicrophoneStore, | ||
| } from "../lib/useMicrophoneStore"; | ||
|
|
||
| /** | ||
| * Records the user and then does something with the recording | ||
| */ | ||
| export function Microphone({ | ||
| onSend, | ||
| onTranscription, | ||
| }: { | ||
| onSend: () => void; | ||
| onTranscription: (text: string) => void | Promise<void>; | ||
| }) { | ||
| const isMicOn = useMicrophoneStore((s) => s.isMicOn); | ||
| const isRecording = useMicrophoneStore((s) => s.isRecording); | ||
|
|
||
| const handleFinishRecording = useCallback(() => { | ||
| onSend(); | ||
| const audioBlob = new Blob(useMicrophoneStore.getState().data, { | ||
| type: "audio/mp4", | ||
| }); | ||
|
|
||
| // Base64 encode the blob | ||
| let audioUrl = ""; | ||
| const reader = new FileReader(); | ||
| reader.readAsDataURL(audioBlob); | ||
| reader.onloadend = () => { | ||
| audioUrl = reader.result as string; | ||
|
|
||
| // Send the base64 data to the server | ||
| fetch("/api/prompt/speech-to-text", { | ||
| method: "POST", | ||
| body: JSON.stringify({ audioUrl }), | ||
| headers: { | ||
| "Content-Type": "application/json", | ||
| }, | ||
| }) | ||
| .then((res) => res.text()) | ||
| .then(onTranscription); | ||
| }; | ||
| }, [onSend, onTranscription]); | ||
|
|
||
| const turnOn = useCallback(() => { | ||
| turnOnMicrophone(handleFinishRecording); | ||
| }, [handleFinishRecording]); | ||
|
|
||
| return ( | ||
| <IconButton2 | ||
| size="xs" | ||
| className={cx("!absolute bottom-0 right-0", { | ||
| "!bg-black !text-white": isMicOn && !isRecording, | ||
| "!bg-red-500 !text-white": isMicOn && isRecording, | ||
| })} | ||
| type="button" | ||
| onClick={turnOn} | ||
| onMouseDown={startRecording} | ||
| onMouseUp={stopRecording} | ||
| > | ||
| <MicrophoneIcon size={16} /> | ||
| </IconButton2> | ||
| ); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| import { create } from "zustand"; | ||
| type MicrophoneStore = { | ||
| isMicOn: boolean; | ||
| isRecording: boolean; | ||
| mediaRecorder: MediaRecorder | null; | ||
| data: Blob[]; | ||
| }; | ||
|
|
||
| export const useMicrophoneStore = create<MicrophoneStore>((set) => ({ | ||
| isMicOn: false, | ||
| isRecording: false, | ||
| mediaRecorder: null, | ||
| data: [], | ||
| })); | ||
|
|
||
| export function turnOnMicrophone(onFinish: () => void | Promise<void>) { | ||
| const isMicOn = useMicrophoneStore.getState().isMicOn; | ||
| if (isMicOn) return; | ||
|
|
||
| // start the microphone listening to audio | ||
| navigator.mediaDevices | ||
| .getUserMedia({ audio: true }) | ||
| .then((stream) => { | ||
| const mediaRecorder = new MediaRecorder(stream); | ||
|
|
||
| mediaRecorder.addEventListener("dataavailable", (e) => { | ||
| useMicrophoneStore.setState((state) => ({ | ||
| data: [...state.data, e.data], | ||
| })); | ||
| }); | ||
|
|
||
| mediaRecorder.addEventListener("start", () => { | ||
| useMicrophoneStore.setState({ isRecording: true }); | ||
| }); | ||
|
|
||
| mediaRecorder.addEventListener("stop", () => { | ||
| useMicrophoneStore.setState({ isRecording: false }); | ||
| onFinish(); | ||
| }); | ||
|
|
||
| useMicrophoneStore.setState({ isMicOn: true, mediaRecorder }); | ||
| }) | ||
| .catch(console.error); | ||
| } | ||
|
|
||
| export function startRecording() { | ||
| const mediaRecorder = useMicrophoneStore.getState().mediaRecorder; | ||
| if (!mediaRecorder) return; | ||
|
|
||
| // Clear the data array | ||
| useMicrophoneStore.setState({ data: [] }); | ||
|
|
||
| mediaRecorder.start(); | ||
| } | ||
|
|
||
| export function stopRecording() { | ||
| const mediaRecorder = useMicrophoneStore.getState().mediaRecorder; | ||
| if (!mediaRecorder) return; | ||
|
|
||
| mediaRecorder.stop(); | ||
| } |
Oops, something went wrong.