Streaming (WebSocket)

export NABR_API_KEY="your-api-key"
uv run python/mic_tilawa_client.py --surah 1

export NABR_API_KEY="your-api-key"
cd dart
dart pub get
dart run mic_tilawa_client.dart --surah 1

npm install @msgpack/msgpack

const OPUS_FRAME_SAMPLES = 320; // 20 ms at 16 kHz

// `frame` is exactly 320 s16le samples. `packet` is the Opus packet bytes,
// not a WebM or Ogg blob.
const packet: Uint8Array = encoder.encode(frame);
socket.send(
  encode({
    method: "CHECK_TILAWA_AUDIO",
    audio: packet,
    client_sent_at_ms: Date.now(),
  }),
);

import { useEffect, useRef, useState } from "react";
import { encode, decode } from "@msgpack/msgpack";

// Methods you send to the server
const WSMethods = {
  START_TILAWA_SESSION: "START_TILAWA_SESSION",
  CHECK_TILAWA_AUDIO: "CHECK_TILAWA_AUDIO",
  RESET_TILAWA_SESSION: "RESET_TILAWA_SESSION",
} as const;

// Events you receive from the server
const WSEvents = {
  QRC_CLIENT_CONNECTED: "QRC_CLIENT_CONNECTED",
  TILAWA_SESSION_STARTED: "TILAWA_SESSION_STARTED",
  TILAWA_SESSION_ENDED: "TILAWA_SESSION_ENDED",
  TILAWA_RESULT: "TILAWA_RESULT",
  KEEP_ALIVE: "KEEP_ALIVE",
  QRC_ERROR: "QRC_ERROR",
} as const;

const SAMPLE_RATE = 16000;
const WAV_CHUNK_SAMPLES = 3200; // 200 ms

const WS_URL =
  "wss://api.qurani.ai/v2?api_key=<YOUR_API_KEY>&lang=ar&audio_format=wav";

function pcmToWav(samples: Int16Array): Uint8Array {
  const dataBytes = samples.length * 2;
  const buffer = new ArrayBuffer(44 + dataBytes);
  const view = new DataView(buffer);
  const write = (offset: number, value: string) => {
    for (let i = 0; i < value.length; i++) {
      view.setUint8(offset + i, value.charCodeAt(i));
    }
  };
  write(0, "RIFF");
  view.setUint32(4, 36 + dataBytes, true);
  write(8, "WAVE");
  write(12, "fmt ");
  view.setUint32(16, 16, true);
  view.setUint16(20, 1, true); // PCM
  view.setUint16(22, 1, true); // mono
  view.setUint32(24, SAMPLE_RATE, true);
  view.setUint32(28, SAMPLE_RATE * 2, true);
  view.setUint16(32, 2, true);
  view.setUint16(34, 16, true);
  write(36, "data");
  view.setUint32(40, dataBytes, true);
  new Int16Array(buffer, 44).set(samples);
  return new Uint8Array(buffer);
}

export default function QrcExample() {
  const connection = useRef<WebSocket | null>(null);
  const audioContext = useRef<AudioContext | null>(null);
  const micStream = useRef<MediaStream | null>(null);

  const [connected, setConnected] = useState(false);
  const [recording, setRecording] = useState(false);

  // 1. Connect and listen for messages
  useEffect(() => {
    const socket = new WebSocket(WS_URL);
    socket.binaryType = "arraybuffer";

    socket.addEventListener("open", () => setConnected(true));
    socket.addEventListener("close", () => setConnected(false));

    socket.addEventListener("message", (event: MessageEvent<ArrayBuffer>) => {
      // Every message is MessagePack-encoded binary
      const message = decode(new Uint8Array(event.data)) as {
        event: string;
        [key: string]: unknown;
      };

      switch (message.event) {
        case WSEvents.QRC_CLIENT_CONNECTED:
          // Connection acknowledged — message.websocket_id, message.credits_balance
          break;

        case WSEvents.TILAWA_SESSION_STARTED:
          // Session is ready — message.session_id, message.exit_code (0–8)
          break;

        case WSEvents.TILAWA_SESSION_ENDED:
          // Session ended (reset or surah finished) — message.session_id, message.asr_call_count
          break;

        case WSEvents.KEEP_ALIVE:
          // Server heartbeat every ~15s — message.timestamp, message.websocket_id
          break;

        case WSEvents.TILAWA_RESULT:
          // Realtime feedback on the recitation:
          // message.verse_index / message.word_index  -> current position
          // message.pending_words                     -> heard, not confirmed yet (replace the last list)
          // message.confirmed_words                   -> newly confirmed words
          // message.skipped_words                     -> memorization errors
          // message.substituted_words                 -> another word said in place of a confirmed word
          // message.tajweed_mistakes                  -> tajweed errors (error_code)
          // message.letter_mistakes                   -> phonetic errors (error_code)
          console.log("result", message);
          break;

        case WSEvents.QRC_ERROR:
          console.error(`${message.code}: ${message.message}`);
          break;
      }
    });

    connection.current = socket;
    return () => socket.close();
  }, []);

  // 2. Start the session, then stream microphone audio
  const startRecording = async () => {
    const socket = connection.current;
    if (socket?.readyState !== WebSocket.OPEN) return;

    // Start a tilawa session (send once per session)
    // Audio sent before TILAWA_SESSION_STARTED arrives is dropped, so wait for it in a real app.
    socket.send(
      encode({
        method: WSMethods.START_TILAWA_SESSION,
        chapter_index: 1, // surah number
        verse_index: 1,   // starting verse
        word_index: 1,    // starting word
        hafz_level: 1,    // 1-3
        tajweed_level: 1, // 1: memorization + tajweed and phonetic
      }),
    );

    // Capture 16 kHz mono PCM and send a complete 200 ms WAV every chunk.
    const stream = await navigator.mediaDevices.getUserMedia({
      audio: { channelCount: 1, sampleRate: SAMPLE_RATE },
    });
    micStream.current = stream;
    const context = new AudioContext({ sampleRate: SAMPLE_RATE });
    audioContext.current = context;
    const source = context.createMediaStreamSource(stream);
    const processor = context.createScriptProcessor(4096, 1, 1);
    const pending: number[] = [];

    processor.onaudioprocess = (event) => {
      if (connection.current?.readyState !== WebSocket.OPEN) return;
      const input = event.inputBuffer.getChannelData(0);
      for (let i = 0; i < input.length; i++) {
        const sample = Math.max(-1, Math.min(1, input[i]));
        pending.push(sample < 0 ? sample * 0x8000 : sample * 0x7fff);
      }
      while (pending.length >= WAV_CHUNK_SAMPLES) {
        const frame = Int16Array.from(pending.splice(0, WAV_CHUNK_SAMPLES));
        connection.current.send(
          encode({
            method: WSMethods.CHECK_TILAWA_AUDIO,
            audio: pcmToWav(frame),
            client_sent_at_ms: Date.now(),
          }),
        );
      }
    };

    const mute = context.createGain();
    mute.gain.value = 0;
    source.connect(processor);
    processor.connect(mute);
    mute.connect(context.destination);

    setRecording(true);
  };

  // 3. Stop recording and reset the session on the server
  const stopRecording = () => {
    audioContext.current?.close();
    audioContext.current = null;
    micStream.current?.getTracks().forEach((track) => track.stop());
    micStream.current = null;

    if (connection.current?.readyState === WebSocket.OPEN) {
      connection.current.send(
        encode({ method: WSMethods.RESET_TILAWA_SESSION }),
      );
    }

    setRecording(false);
  };

  return (
    <button
      disabled={!connected}
      onClick={recording ? stopRecording : startRecording}
    >
      {recording ? "Stop recording" : "Start recording"}
    </button>
  );
}

type StartTilawaSessionPayload = {
  method: "START_TILAWA_SESSION";
  chapter_index: number;
  verse_index: number;
  word_index: number;
  hafz_level: number; // 1-3
  tajweed_level: number; // 1–5. 1: memorization + tajweed and phonetic. 2–5: memorization only
};

type CheckTilawaAudioPayload = {
  method: "CHECK_TILAWA_AUDIO";
  // Opus: one 20 ms raw packet (320 samples at 16 kHz, mono).
  // WAV: one complete 16 kHz mono 16-bit WAV file.
  audio: Uint8Array;
  client_sent_at_ms?: number;
};

// Signals recording stopped. The server checks the audio it already received,
// then ends the session. Send START_TILAWA_SESSION to begin another.
type ResetTilawaSessionPayload = {
  method: "RESET_TILAWA_SESSION";
};

// Optional. The server answers with a KEEP_ALIVE event that echoes sent_at.
type KeepAlivePayload = {
  method: "KEEP_ALIVE";
  sent_at: number; // client time in ms
};

type QRCClientConnected = {
  event: "QRC_CLIENT_CONNECTED";
  websocket_id: string;
  credits_balance?: number | null;
};

type TilawaSessionStarted = {
  event: "TILAWA_SESSION_STARTED";
  exit_code: number; // 0–8; 6 = success
  session_id: string;
};

type TilawaSessionEnded = {
  event: "TILAWA_SESSION_ENDED";
  session_id: string;
  asr_call_count: number;
};

type QRCWord = {
  chapter: number;
  verse: number;
  word: number;
};

type Mistake = {
  chapter: number;
  verse: number;
  word: number;
  letter: number; // zero-based index in the word's phonemes, not its Arabic characters
  speech_like: string; // the word with tashkeel
  message: string; // localized tip; some include the reason
  error_code?: string | null; // stable id, e.g. "tadeef_extra"
};

type SubstitutedWord = {
  chapter: number; // the expected word
  verse: number;
  word: number;
  said: QRCWord | null; // the Quran word read instead; null if not a Quran word
  speech_like: string; // what was heard, in Arabic letters
  message: string; // localized, e.g. "You read (X) instead of (Y)"
};

type TilawaResult = {
  event: "TILAWA_RESULT";
  exit_code: number; // 0–8; 6 = success, 8 = surah finished
  websocket_id: string; // the session id
  request_id: string;
  processing_time_ms: number; // integer; excludes network time
  completed_at: number; // unix seconds
  client_sent_at_ms: number | null; // echoed from the audio message
  chapter_index: number;
  verse_index: number;
  word_index: number;
  pending_words: QRCWord[]; // full snapshot; replaces the previous one
  confirmed_words: QRCWord[]; // newly confirmed only
  skipped_words: QRCWord[];
  substituted_words: SubstitutedWord[];
  tajweed_mistakes: Mistake[];
  letter_mistakes: Mistake[];
};

type KeepAlive = {
  event: "KEEP_ALIVE";
  websocket_id: string;
  timestamp: number; // unix seconds
  client_sent_at_ms?: number | null;
};

type QRCError = {
  event: "QRC_ERROR";
  websocket_id: string;
  code: string; // see QRC_ERROR codes below
  message: string;
};