#!/bin/bash # test_hotwords.sh - Test hotwords and initial_prompt features # # Usage: # ./tests/test_hotwords.sh # ./tests/test_hotwords.sh http://localhost:9000 temp/Recording\ 4.flac # # The audio should contain domain-specific words that Whisper tends to # misspell (brand names, acronyms, unusual proper nouns). The script runs # three transcriptions and compares the results. set -euo pipefail ASR_URL="${1:-http://localhost:9000}" AUDIO_FILE="${2:-temp/Recording 4.flac}" OUTPUT_DIR="temp/hotwords_test_results" # Colors GREEN='\033[0;32m' YELLOW='\033[1;33m' CYAN='\033[0;36m' RED='\033[0;31m' NC='\033[0m' # No Color # Configurable test values - adjust these for your audio HOTWORDS="Speakr,CTranslate2,PyAnnote,WhisperX" INITIAL_PROMPT="This is a meeting about AI-powered audio transcription tools including Speakr, CTranslate2, and PyAnnote." echo -e "${CYAN}============================================${NC}" echo -e "${CYAN} Hotwords & Initial Prompt Test Suite${NC}" echo -e "${CYAN}============================================${NC}" echo "" echo -e "ASR URL: ${YELLOW}${ASR_URL}${NC}" echo -e "Audio file: ${YELLOW}${AUDIO_FILE}${NC}" echo -e "Hotwords: ${YELLOW}${HOTWORDS}${NC}" echo -e "Initial prompt: ${YELLOW}${INITIAL_PROMPT}${NC}" echo "" # Verify audio file exists if [ ! -f "$AUDIO_FILE" ]; then echo -e "${RED}ERROR: Audio file not found: ${AUDIO_FILE}${NC}" exit 1 fi # Verify ASR endpoint is reachable echo -n "Checking ASR endpoint... " if curl -sf "${ASR_URL}/" > /dev/null 2>&1 || curl -sf "${ASR_URL}/health" > /dev/null 2>&1; then echo -e "${GREEN}OK${NC}" else echo -e "${RED}FAILED${NC}" echo "Cannot reach ASR endpoint at ${ASR_URL}" exit 1 fi # Create output directory mkdir -p "$OUTPUT_DIR" # ============================================================== # Test 1: Baseline (no hints) # ============================================================== echo "" echo -e "${CYAN}--- Test 1: Baseline (no hotwords, no initial_prompt) ---${NC}" echo -n "Transcribing... " BASELINE_FILE="$OUTPUT_DIR/1_baseline.json" curl -sS -X POST "${ASR_URL}/asr?output=json&task=transcribe" \ -F "audio_file=@${AUDIO_FILE}" \ -o "$BASELINE_FILE" BASELINE_TEXT=$(python3 -c " import json d=json.load(open('$BASELINE_FILE')) t=d.get('text','') if isinstance(t, list): t=' '.join(seg.get('text','') for seg in t) print(t[:500]) " 2>/dev/null || echo "PARSE_ERROR") echo -e "${GREEN}Done${NC}" echo -e "Preview: ${BASELINE_TEXT:0:200}..." echo "" # ============================================================== # Test 2: With hotwords only # ============================================================== echo -e "${CYAN}--- Test 2: With hotwords ---${NC}" echo -n "Transcribing... " HOTWORDS_FILE="$OUTPUT_DIR/2_with_hotwords.json" curl -sS -X POST "${ASR_URL}/asr?output=json&task=transcribe&hotwords=${HOTWORDS}" \ -F "audio_file=@${AUDIO_FILE}" \ -o "$HOTWORDS_FILE" HOTWORDS_TEXT=$(python3 -c " import json d=json.load(open('$HOTWORDS_FILE')) t=d.get('text','') if isinstance(t, list): t=' '.join(seg.get('text','') for seg in t) print(t[:500]) " 2>/dev/null || echo "PARSE_ERROR") echo -e "${GREEN}Done${NC}" echo -e "Preview: ${HOTWORDS_TEXT:0:200}..." echo "" # ============================================================== # Test 3: With hotwords + initial_prompt # ============================================================== echo -e "${CYAN}--- Test 3: With hotwords + initial_prompt ---${NC}" echo -n "Transcribing... " BOTH_FILE="$OUTPUT_DIR/3_with_both.json" ENCODED_PROMPT=$(python3 -c "import urllib.parse; print(urllib.parse.quote('$INITIAL_PROMPT'))") curl -sS -X POST "${ASR_URL}/asr?output=json&task=transcribe&hotwords=${HOTWORDS}&initial_prompt=${ENCODED_PROMPT}" \ -F "audio_file=@${AUDIO_FILE}" \ -o "$BOTH_FILE" BOTH_TEXT=$(python3 -c " import json d=json.load(open('$BOTH_FILE')) t=d.get('text','') if isinstance(t, list): t=' '.join(seg.get('text','') for seg in t) print(t[:500]) " 2>/dev/null || echo "PARSE_ERROR") echo -e "${GREEN}Done${NC}" echo -e "Preview: ${BOTH_TEXT:0:200}..." echo "" # ============================================================== # Test 4: With initial_prompt only # ============================================================== echo -e "${CYAN}--- Test 4: With initial_prompt only ---${NC}" echo -n "Transcribing... " PROMPT_FILE="$OUTPUT_DIR/4_with_initial_prompt.json" curl -sS -X POST "${ASR_URL}/asr?output=json&task=transcribe&initial_prompt=${ENCODED_PROMPT}" \ -F "audio_file=@${AUDIO_FILE}" \ -o "$PROMPT_FILE" PROMPT_TEXT=$(python3 -c " import json d=json.load(open('$PROMPT_FILE')) t=d.get('text','') if isinstance(t, list): t=' '.join(seg.get('text','') for seg in t) print(t[:500]) " 2>/dev/null || echo "PARSE_ERROR") echo -e "${GREEN}Done${NC}" echo -e "Preview: ${PROMPT_TEXT:0:200}..." echo "" # ============================================================== # Comparison # ============================================================== echo -e "${CYAN}============================================${NC}" echo -e "${CYAN} Comparison Results${NC}" echo -e "${CYAN}============================================${NC}" echo "" # Check if hotwords appear in outputs python3 << 'PYEOF' import json import os def extract_text(data): """Extract full text from ASR response, handling both string and segment list formats.""" text = data.get("text", "") if isinstance(text, list): return " ".join(seg.get("text", "") for seg in text) return text hotwords = ["Speakr", "CTranslate2", "PyAnnote", "WhisperX"] output_dir = os.environ.get("OUTPUT_DIR", "temp/hotwords_test_results") test_files = { "1. Baseline": f"{output_dir}/1_baseline.json", "2. Hotwords only": f"{output_dir}/2_with_hotwords.json", "3. Hotwords + prompt": f"{output_dir}/3_with_both.json", "4. Initial prompt only": f"{output_dir}/4_with_initial_prompt.json", } print(f"{'Test':<25} | {'Hotword Matches':<20} | {'Words Found'}") print("-" * 75) for label, filepath in test_files.items(): try: with open(filepath) as f: data = json.load(f) text = extract_text(data) found = [] for hw in hotwords: if hw.lower() in text.lower(): found.append(hw) match_str = f"{len(found)}/{len(hotwords)}" found_str = ", ".join(found) if found else "(none)" print(f"{label:<25} | {match_str:<20} | {found_str}") except Exception as e: print(f"{label:<25} | ERROR: {e}") print() print("Full outputs saved to: " + output_dir) PYEOF echo "" echo -e "${CYAN}============================================${NC}" echo -e "${CYAN} Precedence Test via Speakr API${NC}" echo -e "${CYAN}============================================${NC}" echo "" echo -e "${YELLOW}To test the full precedence chain (user → folder → tag → upload form),${NC}" echo -e "${YELLOW}use the Speakr web API with authentication:${NC}" echo "" echo -e "1. Set user-level defaults in Account Settings → Prompt Options" echo -e "2. Create a tag with different hotwords/initial_prompt" echo -e "3. Create a folder with different hotwords/initial_prompt" echo -e "4. Upload via API and check server logs for resolved values:" echo "" cat << 'EXAMPLE' # Upload with user defaults only (no tag, no folder) curl -X POST "https://your-speakr/upload" \ -H "Authorization: Bearer YOUR_TOKEN" \ -F "file=@test.flac" # → Should use user defaults # Upload with a tag that has hotwords set curl -X POST "https://your-speakr/upload" \ -H "Authorization: Bearer YOUR_TOKEN" \ -F "file=@test.flac" \ -F "tags=TAG_ID_WITH_HOTWORDS" # → Should use tag defaults (overrides user) # Upload with explicit form values (highest priority) curl -X POST "https://your-speakr/upload" \ -H "Authorization: Bearer YOUR_TOKEN" \ -F "file=@test.flac" \ -F "tags=TAG_ID_WITH_HOTWORDS" \ -F "hotwords=FormOverride1,FormOverride2" \ -F "initial_prompt=Form level prompt" # → Should use form values (overrides tag and user) EXAMPLE echo "" echo -e "${GREEN}Test complete!${NC}"