diff --git a/apps/api/internal/httpapi/exercise_audio_test.go b/apps/api/internal/httpapi/exercise_audio_test.go index ad1f1be..ac088bd 100644 --- a/apps/api/internal/httpapi/exercise_audio_test.go +++ b/apps/api/internal/httpapi/exercise_audio_test.go @@ -34,10 +34,13 @@ type audioSpeechService struct { locale string voice string text string + mode string + word string } -func (s *audioSpeechService) Synthesize(_ context.Context, locale, voice, text string) ([]byte, string, error) { +func (s *audioSpeechService) SynthesizeExercise(_ context.Context, locale, voice, text, mode, word string) ([]byte, string, error) { s.locale, s.voice, s.text = locale, voice, text + s.mode, s.word = mode, word return []byte("mp3-data"), "audio/mpeg", nil } @@ -58,11 +61,50 @@ func TestExerciseAudioUsesStoredSentenceAndLanguageVoice(t *testing.T) { if recorder.Header().Get("Content-Type") != "audio/mpeg" { t.Fatalf("content type = %q", recorder.Header().Get("Content-Type")) } - if lookup.requestedUserID != "usr_current" || speech.locale != "en-US" || speech.voice != "en-US-JennyNeural" || speech.text != "Please pass the water." { + if lookup.requestedUserID != "usr_current" || speech.locale != "en-US" || speech.voice != "en-US-JennyNeural" || speech.text != "Please pass the water." || speech.mode != "normal" { t.Fatalf("unexpected synthesis input: user %q, locale %q, voice %q, text %q", lookup.requestedUserID, speech.locale, speech.voice, speech.text) } } +func TestExerciseAudioSlowAndFocusModes(t *testing.T) { + for _, test := range []struct { + name string + url string + mode string + word string + }{ + {"slow", "/practice/exercises/ex_1/audio?mode=slow", "slow", ""}, + {"focus", "/practice/exercises/ex_1/audio?mode=focus&word=water", "focus", "water"}, + {"focus punctuation", "/practice/exercises/ex_1/audio?mode=focus&word=water.", "focus", "water"}, + } { + t.Run(test.name, func(t *testing.T) { + speech := &audioSpeechService{} + server := &Server{store: &audioExerciseStore{}, speech: speech} + req := httptest.NewRequest(http.MethodGet, test.url, nil) + req.SetPathValue("id", "ex_1") + req = req.WithContext(context.WithValue(req.Context(), userIDKey, "usr_current")) + recorder := httptest.NewRecorder() + server.exerciseAudio(recorder, req) + if recorder.Code != http.StatusOK || speech.mode != test.mode || speech.word != test.word { + t.Fatalf("status = %d, mode = %q, word = %q", recorder.Code, speech.mode, speech.word) + } + }) + } +} + +func TestExerciseAudioRejectsWordsOutsideSentence(t *testing.T) { + speech := &audioSpeechService{} + server := &Server{store: &audioExerciseStore{}, speech: speech} + req := httptest.NewRequest(http.MethodGet, "/practice/exercises/ex_1/audio?mode=focus&word=unrelated", nil) + req.SetPathValue("id", "ex_1") + req = req.WithContext(context.WithValue(req.Context(), userIDKey, "usr_current")) + recorder := httptest.NewRecorder() + server.exerciseAudio(recorder, req) + if recorder.Code != http.StatusBadRequest || speech.text != "" { + t.Fatalf("status = %d, synthesis text = %q", recorder.Code, speech.text) + } +} + func TestExerciseAudioRejectsInaccessibleExercise(t *testing.T) { lookup := &audioExerciseStore{lookupErr: store.ErrNotFound} speech := &audioSpeechService{} diff --git a/apps/api/internal/httpapi/server.go b/apps/api/internal/httpapi/server.go index 4e731f4..1af61b8 100644 --- a/apps/api/internal/httpapi/server.go +++ b/apps/api/internal/httpapi/server.go @@ -9,6 +9,7 @@ import ( "net/http" "strings" "time" + "unicode" "speech-learning-api/internal/config" "speech-learning-api/internal/domain" @@ -306,12 +307,31 @@ func (s *Server) exerciseAudio(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusNotFound, err) return } + mode := r.URL.Query().Get("mode") + if mode == "" { + mode = "normal" + } + if mode != "normal" && mode != "slow" && mode != "focus" { + writeError(w, http.StatusBadRequest, errors.New("unsupported audio mode")) + return + } + focusWord := r.URL.Query().Get("word") + if mode == "focus" { + focusWord = exerciseFocusWord(exercise.TargetText, focusWord) + if focusWord == "" { + writeError(w, http.StatusBadRequest, errors.New("word must occur in the exercise sentence")) + return + } + } else if focusWord != "" { + writeError(w, http.StatusBadRequest, errors.New("word is only supported in focus mode")) + return + } language, err := s.store.GetLanguage(r.Context(), exercise.TargetLanguage) if err != nil { writeError(w, http.StatusInternalServerError, err) return } - audio, mime, err := s.speech.Synthesize(r.Context(), language.AzureLocale, language.DefaultTTSVoice, exercise.TargetText) + audio, mime, err := s.speech.SynthesizeExercise(r.Context(), language.AzureLocale, language.DefaultTTSVoice, exercise.TargetText, mode, focusWord) if err != nil { writeError(w, http.StatusBadGateway, err) return @@ -326,6 +346,20 @@ func (s *Server) exerciseAudio(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(audio) } +func exerciseFocusWord(sentence, requested string) string { + requested = strings.TrimFunc(strings.TrimSpace(requested), func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsNumber(r) }) + if requested == "" { + return "" + } + for _, token := range strings.Fields(sentence) { + word := strings.TrimFunc(token, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsNumber(r) }) + if strings.EqualFold(word, requested) { + return word + } + } + return "" +} + func (s *Server) createAttempt(w http.ResponseWriter, r *http.Request) { if err := r.ParseMultipartForm(24 << 20); err != nil { writeError(w, http.StatusBadRequest, err) diff --git a/apps/api/internal/services/speech.go b/apps/api/internal/services/speech.go index 1325d75..e3c172b 100644 --- a/apps/api/internal/services/speech.go +++ b/apps/api/internal/services/speech.go @@ -22,6 +22,7 @@ import ( type SpeechService interface { AssessPronunciation(ctx context.Context, input PronunciationInput) (domain.PronunciationResult, error) Synthesize(ctx context.Context, locale, voice, text string) ([]byte, string, error) + SynthesizeExercise(ctx context.Context, locale, voice, text, mode, focusWord string) ([]byte, string, error) } type PronunciationInput struct { @@ -98,13 +99,41 @@ func (s *AzureSpeechService) Synthesize(ctx context.Context, locale, voice, text if strings.TrimSpace(text) == "" { return nil, "", errors.New("tts text is required") } - if s.cfg.AzureMock || s.cfg.AzureSpeechKey == "" || s.cfg.AzureRegion == "" { - return nil, "", nil - } - locale = valueOr(locale, "en-US") voice = valueOr(voice, "en-US-JennyNeural") ssml := fmt.Sprintf(`%s`, locale, voice, xmlEscape(text)) + return s.synthesizeSSML(ctx, ssml) +} + +func (s *AzureSpeechService) SynthesizeExercise(ctx context.Context, locale, voice, text, mode, focusWord string) ([]byte, string, error) { + if strings.TrimSpace(text) == "" { + return nil, "", errors.New("tts text is required") + } + if mode == "normal" { + return s.Synthesize(ctx, locale, voice, text) + } + locale = valueOr(locale, "en-US") + voice = valueOr(voice, "en-US-JennyNeural") + var content string + switch mode { + case "slow": + content = fmt.Sprintf(`%s`, xmlEscape(text)) + case "focus": + if strings.TrimSpace(focusWord) == "" { + return nil, "", errors.New("focus word is required") + } + content = fmt.Sprintf(`%s%s`, xmlEscape(focusWord), xmlEscape(text)) + default: + return nil, "", errors.New("unsupported audio mode") + } + ssml := fmt.Sprintf(`%s`, locale, voice, content) + return s.synthesizeSSML(ctx, ssml) +} + +func (s *AzureSpeechService) synthesizeSSML(ctx context.Context, ssml string) ([]byte, string, error) { + if s.cfg.AzureMock || s.cfg.AzureSpeechKey == "" || s.cfg.AzureRegion == "" { + return nil, "", nil + } endpoint := fmt.Sprintf("https://%s.tts.speech.microsoft.com/cognitiveservices/v1", s.cfg.AzureRegion) req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(ssml)) diff --git a/apps/api/internal/services/speech_test.go b/apps/api/internal/services/speech_test.go index 533d261..134cc62 100644 --- a/apps/api/internal/services/speech_test.go +++ b/apps/api/internal/services/speech_test.go @@ -1,6 +1,72 @@ package services -import "testing" +import ( + "context" + "encoding/xml" + "io" + "net/http" + "strings" + "testing" + + "speech-learning-api/internal/config" +) + +type speechRoundTripFunc func(*http.Request) (*http.Response, error) + +func (fn speechRoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} + +func TestSynthesizeExerciseSendsValidSSML(t *testing.T) { + for _, test := range []struct { + mode string + word string + rates []string + }{ + {"slow", "", []string{"0.8"}}, + {"focus", "chips", []string{"0.7", "0.8"}}, + } { + t.Run(test.mode, func(t *testing.T) { + var body string + service := &AzureSpeechService{ + cfg: config.Config{AzureSpeechKey: "test-key", AzureRegion: "test-region"}, + client: &http.Client{Transport: speechRoundTripFunc(func(req *http.Request) (*http.Response, error) { + payload, err := io.ReadAll(req.Body) + if err != nil { + t.Fatal(err) + } + body = string(payload) + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader("mp3")), Header: make(http.Header)}, nil + })}, + } + _, _, err := service.SynthesizeExercise(context.Background(), "en-US", "en-US-JennyNeural", "Fish & chips are tasty.", test.mode, test.word) + if err != nil { + t.Fatal(err) + } + decoder := xml.NewDecoder(strings.NewReader(body)) + for { + _, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + t.Fatalf("invalid SSML: %v; body %q", err, body) + } + } + if !strings.Contains(body, "Fish & chips are tasty.") { + t.Fatalf("sentence was not XML-escaped: %q", body) + } + for _, rate := range test.rates { + if !strings.Contains(body, `rate="`+rate+`"`) { + t.Fatalf("missing rate %s: %q", rate, body) + } + } + if test.mode == "focus" && (!strings.Contains(body, ``) || !strings.Contains(body, `>chips`)) { + t.Fatalf("focus clip lacks the word or pause: %q", body) + } + }) + } +} func TestMockPronunciationReturnsWordScores(t *testing.T) { result := mockPronunciation("Hello, my name is Anna.") diff --git a/apps/web/src/api/lessons.ts b/apps/web/src/api/lessons.ts index 95284d1..19f43a5 100644 --- a/apps/web/src/api/lessons.ts +++ b/apps/web/src/api/lessons.ts @@ -16,8 +16,14 @@ export function fetchProgress() { return request('/progress'); } -export function fetchExerciseAudio(exerciseId: string, signal?: AbortSignal) { - return requestBlob(`/practice/exercises/${encodeURIComponent(exerciseId)}/audio`, { signal }); +export type ExerciseAudioMode = 'normal' | 'slow' | 'focus'; + +export function fetchExerciseAudio(exerciseId: string, mode: ExerciseAudioMode = 'normal', word = '', signal?: AbortSignal) { + const params = new URLSearchParams(); + if (mode !== 'normal') params.set('mode', mode); + if (mode === 'focus') params.set('word', word); + const query = params.size ? `?${params.toString()}` : ''; + return requestBlob(`/practice/exercises/${encodeURIComponent(exerciseId)}/audio${query}`, { signal }); } export async function fetchNextLesson(fresh = false, recentExerciseIds: string[] = []) { diff --git a/apps/web/src/pages/PracticePage.tsx b/apps/web/src/pages/PracticePage.tsx index 7facce1..16effd8 100644 --- a/apps/web/src/pages/PracticePage.tsx +++ b/apps/web/src/pages/PracticePage.tsx @@ -1,8 +1,8 @@ import { RotateCcw, Send, Volume2 } from 'lucide-react'; import { useEffect, useRef, useState } from 'react'; -import { fetchExerciseAudio, fetchNextLesson } from '../api/lessons'; +import { fetchExerciseAudio, fetchNextLesson, type ExerciseAudioMode } from '../api/lessons'; import { submitAttempt } from '../api/practice'; -import type { AttemptResponse, Exercise, Lesson } from '../api/types'; +import type { AttemptResponse, Exercise, Lesson, WordScore } from '../api/types'; import { AudioRecorder } from '../components/AudioRecorder'; import { PhonemeAnalysis } from '../components/PhonemeAnalysis'; import { PronunciationText } from '../components/PronunciationText'; @@ -21,12 +21,12 @@ export function PracticePage({ initialSelection }: Props) { const [loading, setLoading] = useState(false); const [loadingNext, setLoadingNext] = useState(false); const [recording, setRecording] = useState(false); - const [audioLoading, setAudioLoading] = useState(false); + const [audioLoadingKey, setAudioLoadingKey] = useState(null); const [audioError, setAudioError] = useState(''); const [error, setError] = useState(''); const [notice, setNotice] = useState(''); const submittingRef = useRef(false); - const referenceAudioRef = useRef<{ exerciseId: string; url: string; player: HTMLAudioElement } | null>(null); + const referenceAudioRef = useRef<{ exerciseId: string; urls: Map; player: HTMLAudioElement | null } | null>(null); const audioRequestRef = useRef(null); const audioGenerationRef = useRef(0); const recentExerciseIds = useRef(initialSelection?.exercise.id ? [initialSelection.exercise.id] : []); @@ -49,35 +49,41 @@ export function PracticePage({ initialSelection }: Props) { audioGenerationRef.current += 1; audioRequestRef.current?.abort(); audioRequestRef.current = null; - referenceAudioRef.current?.player.pause(); - if (referenceAudioRef.current) URL.revokeObjectURL(referenceAudioRef.current.url); + referenceAudioRef.current?.player?.pause(); + referenceAudioRef.current?.urls.forEach((url) => URL.revokeObjectURL(url)); referenceAudioRef.current = null; }, [exercise?.id]); - async function playReference() { - if (!exercise || audioLoading || audioRequestRef.current || recording) return; + async function playReference(mode: ExerciseAudioMode, word = '') { + if (!exercise || audioLoadingKey || audioRequestRef.current || recording) return; setAudioError(''); const exerciseId = exercise.id; + const audioKey = `${mode}:${word.toLocaleLowerCase()}`; const generation = audioGenerationRef.current; let controller: AbortController | null = null; try { if (!referenceAudioRef.current || referenceAudioRef.current.exerciseId !== exerciseId) { - setAudioLoading(true); + referenceAudioRef.current = { exerciseId, urls: new Map(), player: null }; + } + let url = referenceAudioRef.current.urls.get(audioKey); + if (!url) { + setAudioLoadingKey(audioKey); controller = new AbortController(); audioRequestRef.current = controller; - const blob = await fetchExerciseAudio(exerciseId, controller.signal); + const blob = await fetchExerciseAudio(exerciseId, mode, word, controller.signal); if (controller.signal.aborted || audioGenerationRef.current !== generation) return; - const url = URL.createObjectURL(blob); - referenceAudioRef.current = { exerciseId, url, player: new Audio(url) }; + url = URL.createObjectURL(blob); + referenceAudioRef.current.urls.set(audioKey, url); } - const player = referenceAudioRef.current.player; - player.currentTime = 0; + referenceAudioRef.current.player?.pause(); + const player = new Audio(url); + referenceAudioRef.current.player = player; await player.play(); } catch { if (audioGenerationRef.current !== generation || controller?.signal.aborted) return; setAudioError('Satz konnte nicht abgespielt werden. Bitte erneut versuchen.'); } finally { - if (audioGenerationRef.current === generation) setAudioLoading(false); + if (audioGenerationRef.current === generation) setAudioLoadingKey(null); if (audioRequestRef.current === controller) audioRequestRef.current = null; } } @@ -87,8 +93,8 @@ export function PracticePage({ initialSelection }: Props) { audioGenerationRef.current += 1; audioRequestRef.current?.abort(); audioRequestRef.current = null; - referenceAudioRef.current?.player.pause(); - setAudioLoading(false); + referenceAudioRef.current?.player?.pause(); + setAudioLoadingKey(null); setLoadingNext(true); setError(''); setNotice(''); @@ -137,6 +143,8 @@ export function PracticePage({ initialSelection }: Props) { ); } + const focusWords = result ? practiceFocusWords(exercise.target_text, result.attempt.word_scores) : []; + return (
@@ -155,9 +163,26 @@ export function PracticePage({ initialSelection }: Props) {

{exercise.prompt_text}

- +
+ + +
+ {focusWords.length > 0 && ( +
+

Diese Wörter kannst du gezielt anhören:

+
+ {focusWords.map((word) => ( + + ))} +
+
+ )} {audioError &&

{audioError}

}
); } + +function practiceFocusWords(sentence: string, scores: WordScore[]): string[] { + const sentenceWords = sentence.split(/\s+/).map(cleanWord); + const seen = new Set(); + return scores + .map((score, index) => ({ + word: sentenceWords.find((word) => word.toLocaleLowerCase() === cleanWord(score.word).toLocaleLowerCase()) ?? '', + score: Math.min(score.accuracy, ...(score.phoneme_scores ?? []).map((phoneme) => phoneme.accuracy)), + index + })) + .filter((item) => item.word && item.score < 75) + .sort((a, b) => a.score - b.score || a.index - b.index) + .filter((item) => { + const key = item.word.toLocaleLowerCase(); + if (seen.has(key)) return false; + seen.add(key); + return true; + }) + .slice(0, 3) + .map((item) => item.word); +} + +function cleanWord(value: string): string { + return value.replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, ''); +} diff --git a/docs/api.md b/docs/api.md index ebbf856..cf37fc4 100644 --- a/docs/api.md +++ b/docs/api.md @@ -32,6 +32,8 @@ Authorization: Bearer - `GET /practice/exercises/{id}/audio` (Bearer-Token erforderlich; liefert den Satz der zugänglichen Übung als `audio/mpeg`) - `POST /practice/attempts` +`GET /practice/exercises/{id}/audio` unterstützt `mode=normal` (Standard), `mode=slow` und `mode=focus&word=water`. Im Fokusmodus spricht Azure zuerst das Wort, dann nach einer Pause den Satz langsam. Das Wort muss im gespeicherten Übungssatz vorkommen. + `POST /practice/attempts` expects multipart form data: - `exercise_id`