app-santa-cruz.js

A developer built a real-time voice assistant for the 'Build with AI de Santa Cruz' hackathon using Google's Gemini 3.1 Flash Live Preview model. The assistant, named Jarvis, processes audio input and output via Web Audio API and communicates with the Gemini API to provide conversational guidance to student participants.

| import { GoogleGenAI, Modality } from "https://esm.sh/@google/genai"; | | | import { key } from './ key.js' | | | const API KEY = key; | | | let audioContext; | | | let nextStartTime = 0; | | | const audioSources = new Set ; | | | let audioInputContext; | | | let micStream; | | | let audioProcessor; | | | const genAI = new GoogleGenAI { | | | apiKey: API KEY, | | | } ; | | | const instructions = | | | Eres el asistente oficial del evento 'Build with AI de Santa Cruz'. Tu propósito es guiar, inspirar y ayudar a los estudiantes de la universidad que participan en esta hackathon. Al iniciar la conexión, debes saludar con entusiasmo, darles la bienvenida formal al 'Build with AI de Santa Cruz', explicar brevemente quién eres el copiloto de IA de la hackathon y preguntarles si tienen alguna duda o idea que quieran empezar a pelotear. | | | REGLA CRÍTICA PARA VOZ: Mantén tus respuestas muy cortas, conversacionales y directas máximo 2 o 3 oraciones por intervención . No uses listas con viñetas ni texto que suene raro al ser leído en voz alta. Habla con un tono amigable, tecnológico y motivador | | | ; | | | const config = { | | | responseModalities: Modality.AUDIO , | | | systemInstruction: instructions, | | | outputAudioTranscription: {}, | | | inputAudioTranscription: {}, | | | }; | | | const session = await genAI.live.connect { | | | model: 'gemini-3.1-flash-live-preview', | | | config: config, | | | callbacks: { | | | onmessage: async message = { | | | const { setupComplete, serverContent } = message; | | | if setupComplete { | | | console.log 'Conectado' ; | | | sendTextToIa "Empieza a hablar" ; | | | startMic ; | | | } | | | if serverContent?.outputTranscription { | | | const newText = serverContent.outputTranscription.text; | | | console.log Jarvis dice: ${newText} ; | | | } | | | if serverContent?.inputTranscription { | | | const newText = serverContent.inputTranscription.text; | | | console.log User dice: ${newText} ; | | | } | | | const audio = serverContent?.modelTurn?.parts 0 ?.inlineData; | | | if audio { | | | playAIAudio audio ; | | | } | | | }, | | | onerror: async e = { | | | console.error 'Session Error:', e ; | | | }, | | | onclose: async e = { | | | console.warn 'Session Closed:', e ; | | | }, | | | }, | | | } ; | | | window.sendTextToIa = text = { | | | session.sendRealtimeInput { | | | text: text, | | | } ; | | | } | | | function playAIAudio audio { | | | if audioContext { | | | audioContext = new window.AudioContext || window.webkitAudioContext { | | | sampleRate: 24000 | | | } ; | | | nextStartTime = audioContext.currentTime; | | | } | | | const rawBytes = base64Decode audio.data ; | | | const aiFloat32 24k = getFloat32FromPCM rawBytes ; | | | nextStartTime = Math.max nextStartTime, audioContext.currentTime ; | | | const audioBuffer = audioContext.createBuffer 1, aiFloat32 24k.length, 24000 ; | | | audioBuffer.getChannelData 0 .set aiFloat32 24k ; | | | const source = audioContext.createBufferSource ; | | | source.buffer = audioBuffer; | | | source.connect audioContext.destination ; | | | source.addEventListener 'ended', = audioSources.delete source ; | | | source.start nextStartTime ; | | | nextStartTime += audioBuffer.duration; | | | audioSources.add source ; | | | } | | | function getFloat32FromPCM bytesCrudos { | | | const dataInt16 = new Int16Array bytesCrudos.buffer ; | | | const dataFloat32 = new Float32Array dataInt16.length ; | | | for let i = 0; i < dataInt16.length; i++ { | | | dataFloat32 i = dataInt16 i / 32768.0; | | | } | | | return dataFloat32; | | | } | | | function base64Decode base64 { | | | const binaryString = atob base64 ; | | | const len = binaryString.length; | | | const bytes = new Uint8Array len ; | | | for let i = 0; i < len; i++ { | | | bytes i = binaryString.charCodeAt i ; | | | } | | | return bytes; | | | } | | | window.startMic = async = { | | | try { | | | micStream = await navigator.mediaDevices.getUserMedia { audio: true } ; | | | audioInputContext = new window.AudioContext || window.webkitAudioContext { | | | sampleRate: 16000 | | | } ; | | | const source = audioInputContext.createMediaStreamSource micStream ; | | | audioProcessor = audioInputContext.createScriptProcessor 4096, 1, 1 ; | | | audioProcessor.onaudioprocess = e = { | | | const float32Data = e.inputBuffer.getChannelData 0 ; | | | const pcm16Data = floatTo16BitPCM float32Data ; | | | const base64Audio = arrayBufferToBase64 pcm16Data ; | | | session.sendRealtimeInput { | | | audio: { | | | data: base64Audio, | | | mimeType: audio/pcm;rate=16000 | | | }, | | | } | | | }; | | | source.connect audioProcessor ; | | | audioProcessor.connect audioInputContext.destination ; | | | console.log 'Micrófono conectado' ; | | | } | | | catch error { | | | console.warn "Mic error:", error ; | | | } | | | } | | | function floatTo16BitPCM float32Array { | | | const buffer = new ArrayBuffer float32Array.length 2 ; | | | const view = new DataView buffer ; | | | let offset = 0; | | | for let i = 0; i < float32Array.length; i++, offset += 2 { | | | let s = Math.max -1, Math.min 1, float32Array i ; | | | view.setInt16 offset, s < 0 ? s 0x8000 : s 0x7FFF, true ; | | | } | | | return buffer; | | | } | | | function arrayBufferToBase64 buffer { | | | let binary = ''; | | | const bytes = new Uint8Array buffer ; | | | const len = bytes.byteLength; | | | for let i = 0; i < len; i++ { | | | binary += String.fromCharCode bytes i ; | | | } | | | return window.btoa binary ; | | | } |