Initial release: DictIA v0.8.14-alpha (fork de Speakr, AGPL-3.0)
This commit is contained in:
InnovA AI
241
tests/test_hotwords.sh
Executable file
241
tests/test_hotwords.sh
Executable file
@@ -0,0 +1,241 @@
|
||||
#!/bin/bash
|
||||
# test_hotwords.sh - Test hotwords and initial_prompt features
|
||||
#
|
||||
# Usage:
|
||||
# ./tests/test_hotwords.sh <asr_url> <audio_file>
|
||||
# ./tests/test_hotwords.sh http://localhost:9000 temp/Recording\ 4.flac
|
||||
#
|
||||
# The audio should contain domain-specific words that Whisper tends to
|
||||
# misspell (brand names, acronyms, unusual proper nouns). The script runs
|
||||
# three transcriptions and compares the results.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ASR_URL="${1:-http://localhost:9000}"
|
||||
AUDIO_FILE="${2:-temp/Recording 4.flac}"
|
||||
OUTPUT_DIR="temp/hotwords_test_results"
|
||||
|
||||
# Colors
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
RED='\033[0;31m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Configurable test values - adjust these for your audio
|
||||
HOTWORDS="Speakr,CTranslate2,PyAnnote,WhisperX"
|
||||
INITIAL_PROMPT="This is a meeting about AI-powered audio transcription tools including Speakr, CTranslate2, and PyAnnote."
|
||||
|
||||
echo -e "${CYAN}============================================${NC}"
|
||||
echo -e "${CYAN} Hotwords & Initial Prompt Test Suite${NC}"
|
||||
echo -e "${CYAN}============================================${NC}"
|
||||
echo ""
|
||||
echo -e "ASR URL: ${YELLOW}${ASR_URL}${NC}"
|
||||
echo -e "Audio file: ${YELLOW}${AUDIO_FILE}${NC}"
|
||||
echo -e "Hotwords: ${YELLOW}${HOTWORDS}${NC}"
|
||||
echo -e "Initial prompt: ${YELLOW}${INITIAL_PROMPT}${NC}"
|
||||
echo ""
|
||||
|
||||
# Verify audio file exists
|
||||
if [ ! -f "$AUDIO_FILE" ]; then
|
||||
echo -e "${RED}ERROR: Audio file not found: ${AUDIO_FILE}${NC}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Verify ASR endpoint is reachable
|
||||
echo -n "Checking ASR endpoint... "
|
||||
if curl -sf "${ASR_URL}/" > /dev/null 2>&1 || curl -sf "${ASR_URL}/health" > /dev/null 2>&1; then
|
||||
echo -e "${GREEN}OK${NC}"
|
||||
else
|
||||
echo -e "${RED}FAILED${NC}"
|
||||
echo "Cannot reach ASR endpoint at ${ASR_URL}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create output directory
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
|
||||
# ==============================================================
|
||||
# Test 1: Baseline (no hints)
|
||||
# ==============================================================
|
||||
echo ""
|
||||
echo -e "${CYAN}--- Test 1: Baseline (no hotwords, no initial_prompt) ---${NC}"
|
||||
echo -n "Transcribing... "
|
||||
|
||||
BASELINE_FILE="$OUTPUT_DIR/1_baseline.json"
|
||||
curl -sS -X POST "${ASR_URL}/asr?output=json&task=transcribe" \
|
||||
-F "audio_file=@${AUDIO_FILE}" \
|
||||
-o "$BASELINE_FILE"
|
||||
|
||||
BASELINE_TEXT=$(python3 -c "
|
||||
import json
|
||||
d=json.load(open('$BASELINE_FILE'))
|
||||
t=d.get('text','')
|
||||
if isinstance(t, list):
|
||||
t=' '.join(seg.get('text','') for seg in t)
|
||||
print(t[:500])
|
||||
" 2>/dev/null || echo "PARSE_ERROR")
|
||||
echo -e "${GREEN}Done${NC}"
|
||||
echo -e "Preview: ${BASELINE_TEXT:0:200}..."
|
||||
echo ""
|
||||
|
||||
# ==============================================================
|
||||
# Test 2: With hotwords only
|
||||
# ==============================================================
|
||||
echo -e "${CYAN}--- Test 2: With hotwords ---${NC}"
|
||||
echo -n "Transcribing... "
|
||||
|
||||
HOTWORDS_FILE="$OUTPUT_DIR/2_with_hotwords.json"
|
||||
curl -sS -X POST "${ASR_URL}/asr?output=json&task=transcribe&hotwords=${HOTWORDS}" \
|
||||
-F "audio_file=@${AUDIO_FILE}" \
|
||||
-o "$HOTWORDS_FILE"
|
||||
|
||||
HOTWORDS_TEXT=$(python3 -c "
|
||||
import json
|
||||
d=json.load(open('$HOTWORDS_FILE'))
|
||||
t=d.get('text','')
|
||||
if isinstance(t, list):
|
||||
t=' '.join(seg.get('text','') for seg in t)
|
||||
print(t[:500])
|
||||
" 2>/dev/null || echo "PARSE_ERROR")
|
||||
echo -e "${GREEN}Done${NC}"
|
||||
echo -e "Preview: ${HOTWORDS_TEXT:0:200}..."
|
||||
echo ""
|
||||
|
||||
# ==============================================================
|
||||
# Test 3: With hotwords + initial_prompt
|
||||
# ==============================================================
|
||||
echo -e "${CYAN}--- Test 3: With hotwords + initial_prompt ---${NC}"
|
||||
echo -n "Transcribing... "
|
||||
|
||||
BOTH_FILE="$OUTPUT_DIR/3_with_both.json"
|
||||
ENCODED_PROMPT=$(python3 -c "import urllib.parse; print(urllib.parse.quote('$INITIAL_PROMPT'))")
|
||||
curl -sS -X POST "${ASR_URL}/asr?output=json&task=transcribe&hotwords=${HOTWORDS}&initial_prompt=${ENCODED_PROMPT}" \
|
||||
-F "audio_file=@${AUDIO_FILE}" \
|
||||
-o "$BOTH_FILE"
|
||||
|
||||
BOTH_TEXT=$(python3 -c "
|
||||
import json
|
||||
d=json.load(open('$BOTH_FILE'))
|
||||
t=d.get('text','')
|
||||
if isinstance(t, list):
|
||||
t=' '.join(seg.get('text','') for seg in t)
|
||||
print(t[:500])
|
||||
" 2>/dev/null || echo "PARSE_ERROR")
|
||||
echo -e "${GREEN}Done${NC}"
|
||||
echo -e "Preview: ${BOTH_TEXT:0:200}..."
|
||||
echo ""
|
||||
|
||||
# ==============================================================
|
||||
# Test 4: With initial_prompt only
|
||||
# ==============================================================
|
||||
echo -e "${CYAN}--- Test 4: With initial_prompt only ---${NC}"
|
||||
echo -n "Transcribing... "
|
||||
|
||||
PROMPT_FILE="$OUTPUT_DIR/4_with_initial_prompt.json"
|
||||
curl -sS -X POST "${ASR_URL}/asr?output=json&task=transcribe&initial_prompt=${ENCODED_PROMPT}" \
|
||||
-F "audio_file=@${AUDIO_FILE}" \
|
||||
-o "$PROMPT_FILE"
|
||||
|
||||
PROMPT_TEXT=$(python3 -c "
|
||||
import json
|
||||
d=json.load(open('$PROMPT_FILE'))
|
||||
t=d.get('text','')
|
||||
if isinstance(t, list):
|
||||
t=' '.join(seg.get('text','') for seg in t)
|
||||
print(t[:500])
|
||||
" 2>/dev/null || echo "PARSE_ERROR")
|
||||
echo -e "${GREEN}Done${NC}"
|
||||
echo -e "Preview: ${PROMPT_TEXT:0:200}..."
|
||||
echo ""
|
||||
|
||||
# ==============================================================
|
||||
# Comparison
|
||||
# ==============================================================
|
||||
echo -e "${CYAN}============================================${NC}"
|
||||
echo -e "${CYAN} Comparison Results${NC}"
|
||||
echo -e "${CYAN}============================================${NC}"
|
||||
echo ""
|
||||
|
||||
# Check if hotwords appear in outputs
|
||||
python3 << 'PYEOF'
|
||||
import json
|
||||
import os
|
||||
|
||||
def extract_text(data):
|
||||
"""Extract full text from ASR response, handling both string and segment list formats."""
|
||||
text = data.get("text", "")
|
||||
if isinstance(text, list):
|
||||
return " ".join(seg.get("text", "") for seg in text)
|
||||
return text
|
||||
|
||||
hotwords = ["Speakr", "CTranslate2", "PyAnnote", "WhisperX"]
|
||||
output_dir = os.environ.get("OUTPUT_DIR", "temp/hotwords_test_results")
|
||||
test_files = {
|
||||
"1. Baseline": f"{output_dir}/1_baseline.json",
|
||||
"2. Hotwords only": f"{output_dir}/2_with_hotwords.json",
|
||||
"3. Hotwords + prompt": f"{output_dir}/3_with_both.json",
|
||||
"4. Initial prompt only": f"{output_dir}/4_with_initial_prompt.json",
|
||||
}
|
||||
|
||||
print(f"{'Test':<25} | {'Hotword Matches':<20} | {'Words Found'}")
|
||||
print("-" * 75)
|
||||
|
||||
for label, filepath in test_files.items():
|
||||
try:
|
||||
with open(filepath) as f:
|
||||
data = json.load(f)
|
||||
text = extract_text(data)
|
||||
found = []
|
||||
for hw in hotwords:
|
||||
if hw.lower() in text.lower():
|
||||
found.append(hw)
|
||||
match_str = f"{len(found)}/{len(hotwords)}"
|
||||
found_str = ", ".join(found) if found else "(none)"
|
||||
print(f"{label:<25} | {match_str:<20} | {found_str}")
|
||||
except Exception as e:
|
||||
print(f"{label:<25} | ERROR: {e}")
|
||||
|
||||
print()
|
||||
print("Full outputs saved to: " + output_dir)
|
||||
PYEOF
|
||||
|
||||
echo ""
|
||||
echo -e "${CYAN}============================================${NC}"
|
||||
echo -e "${CYAN} Precedence Test via Speakr API${NC}"
|
||||
echo -e "${CYAN}============================================${NC}"
|
||||
echo ""
|
||||
echo -e "${YELLOW}To test the full precedence chain (user → folder → tag → upload form),${NC}"
|
||||
echo -e "${YELLOW}use the Speakr web API with authentication:${NC}"
|
||||
echo ""
|
||||
echo -e "1. Set user-level defaults in Account Settings → Prompt Options"
|
||||
echo -e "2. Create a tag with different hotwords/initial_prompt"
|
||||
echo -e "3. Create a folder with different hotwords/initial_prompt"
|
||||
echo -e "4. Upload via API and check server logs for resolved values:"
|
||||
echo ""
|
||||
cat << 'EXAMPLE'
|
||||
# Upload with user defaults only (no tag, no folder)
|
||||
curl -X POST "https://your-speakr/upload" \
|
||||
-H "Authorization: Bearer YOUR_TOKEN" \
|
||||
-F "file=@test.flac"
|
||||
# → Should use user defaults
|
||||
|
||||
# Upload with a tag that has hotwords set
|
||||
curl -X POST "https://your-speakr/upload" \
|
||||
-H "Authorization: Bearer YOUR_TOKEN" \
|
||||
-F "file=@test.flac" \
|
||||
-F "tags=TAG_ID_WITH_HOTWORDS"
|
||||
# → Should use tag defaults (overrides user)
|
||||
|
||||
# Upload with explicit form values (highest priority)
|
||||
curl -X POST "https://your-speakr/upload" \
|
||||
-H "Authorization: Bearer YOUR_TOKEN" \
|
||||
-F "file=@test.flac" \
|
||||
-F "tags=TAG_ID_WITH_HOTWORDS" \
|
||||
-F "hotwords=FormOverride1,FormOverride2" \
|
||||
-F "initial_prompt=Form level prompt"
|
||||
# → Should use form values (overrides tag and user)
|
||||
EXAMPLE
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}Test complete!${NC}"
|
||||
Reference in New Issue
Block a user