Phonembewertung speichern und anzeigen

Azure-Phonemwerte in Versuchen sichern, schwache Laute im Fortschritt und Flowise-Kontext auswerten und die Web-Anzeige ergänzen. README dokumentiert Datenfluss, Betrieb und Grenzen der Funktion.
This commit is contained in:
Eugenius Naumenko
2026-10-10 17:59:44 +00:00
parent bac840e091
commit 82d8768a99
13 changed files with 451 additions and 59 deletions
+24 -10
View File
@@ -49,11 +49,24 @@ type Exercise struct {
}
type WordScore struct {
Word string `json:"word"`
Accuracy float64 `json:"accuracy"`
ErrorType string `json:"error_type,omitempty"`
Phonemes []string `json:"phonemes,omitempty"`
Highlight string `json:"highlight"`
Word string `json:"word"`
Accuracy float64 `json:"accuracy"`
ErrorType string `json:"error_type,omitempty"`
Phonemes []string `json:"phonemes,omitempty"`
PhonemeScores []PhonemeScore `json:"phoneme_scores,omitempty"`
Highlight string `json:"highlight"`
}
type PhonemeScore struct {
Phoneme string `json:"phoneme"`
Accuracy float64 `json:"accuracy"`
Highlight string `json:"highlight"`
}
type WeakPhoneme struct {
Word string `json:"word"`
Phoneme string `json:"phoneme"`
Accuracy float64 `json:"accuracy"`
}
type PronunciationResult struct {
@@ -84,11 +97,12 @@ type Attempt struct {
}
type ProgressSummary struct {
TotalAttempts int `json:"total_attempts"`
AverageScore float64 `json:"average_score"`
WeakWords []WordScore `json:"weak_words"`
RecentAttempts []Attempt `json:"recent_attempts"`
DailyGoalMinutes int `json:"daily_goal_minutes"`
TotalAttempts int `json:"total_attempts"`
AverageScore float64 `json:"average_score"`
WeakWords []WordScore `json:"weak_words"`
WeakPhonemes []WeakPhoneme `json:"weak_phonemes"`
RecentAttempts []Attempt `json:"recent_attempts"`
DailyGoalMinutes int `json:"daily_goal_minutes"`
}
type GeneratedExerciseRequest struct {
+42 -5
View File
@@ -54,6 +54,7 @@ func (f *FlowiseTeacher) NextExercise(ctx context.Context, input domain.Generate
ReasonForChoice: "Fallback exercise because Flowise did not return a valid exercise.",
Source: "fallback",
}
progress := compactProgressForFlowise(input.Progress)
payload := map[string]any{
"task": "next_exercise",
"user_id": input.UserID,
@@ -61,16 +62,37 @@ func (f *FlowiseTeacher) NextExercise(ctx context.Context, input domain.Generate
"cefr_level": input.CEFRLevel,
"ui_language": input.UILanguage,
"profile": input.Profile,
"progress": input.Progress,
"recent_weak_words": input.RecentWeakWords,
"progress": progress,
"recent_weak_words": withoutPhonemeScores(input.RecentWeakWords),
"recent_weak_phonemes": input.Progress.WeakPhonemes,
"preferred_type": valueOr(input.PreferredType, "repeat_sentence"),
"avoid_reference_text": input.AvoidReferenceText,
"avoid_reference_texts": input.AvoidReferenceTexts,
"rules": "Generate one A1 English pronunciation exercise. Return valid JSON only with fields: exercise_type, target_language, cefr_level, reference_text, display_prompt, tts_text, focus_points, success_criteria, reason_for_choice. The reference_text must differ from every sentence in avoid_reference_texts. The same target word may still be used.",
"rules": "Generate one A1 English pronunciation exercise. Return valid JSON only with fields: exercise_type, target_language, cefr_level, reference_text, display_prompt, tts_text, focus_points, success_criteria, reason_for_choice. The reference_text must differ from every sentence in avoid_reference_texts. Use recent_weak_phonemes only when measured scores are provided; do not invent pronunciation errors.",
}
return f.askExercise(ctx, payload, input, fallback)
}
func compactProgressForFlowise(progress domain.ProgressSummary) domain.ProgressSummary {
progress.WeakWords = withoutPhonemeScores(progress.WeakWords)
recent := make([]domain.Attempt, len(progress.RecentAttempts))
for i, attempt := range progress.RecentAttempts {
recent[i] = attempt
recent[i].WordScores = withoutPhonemeScores(attempt.WordScores)
}
progress.RecentAttempts = recent
return progress
}
func withoutPhonemeScores(words []domain.WordScore) []domain.WordScore {
clean := make([]domain.WordScore, len(words))
copy(clean, words)
for i := range clean {
clean[i].PhonemeScores = nil
}
return clean
}
func (f *FlowiseTeacher) FeedbackForAttempt(ctx context.Context, targetText string, result domain.PronunciationResult, weakWords []domain.WordScore) (domain.TeacherResponse, error) {
_ = ctx
_ = weakWords
@@ -261,10 +283,22 @@ func parseGeneratedExerciseJSON(value json.RawMessage) (domain.GeneratedExercise
func mockFeedback(targetText string, result domain.PronunciationResult) domain.TeacherResponse {
weakWord := ""
weakPhoneme := domain.PhonemeScore{}
weakPhonemeWord := ""
for _, word := range result.WordScores {
if word.Highlight == "weak" {
weakWord = word.Word
break
if weakWord == "" {
weakWord = word.Word
}
}
if strings.EqualFold(word.ErrorType, "Omission") {
continue
}
for _, phoneme := range word.PhonemeScores {
if phoneme.Accuracy < 75 && (weakPhonemeWord == "" || phoneme.Accuracy < weakPhoneme.Accuracy) {
weakPhoneme = phoneme
weakPhonemeWord = word.Word
}
}
}
feedback := "Der Satz braucht noch etwas Übung. Sprich ihn noch einmal nach."
@@ -273,6 +307,9 @@ func mockFeedback(targetText string, result domain.PronunciationResult) domain.T
case result.OverallScore >= 95 && result.AccuracyScore >= 95 && result.FluencyScore >= 95 && result.CompletenessScore >= 95:
feedback = "Ausgezeichnet gesprochen! Die Aussprache dieses Satzes wurde sehr hoch bewertet."
nextRecommendation = "Du kannst mit einer neuen Übung weitermachen."
case weakPhonemeWord != "":
feedback = fmt.Sprintf("Im Wort %q wurde der Laut /%s/ mit %.0f von 100 Punkten bewertet.", weakPhonemeWord, weakPhoneme.Phoneme, weakPhoneme.Accuracy)
nextRecommendation = fmt.Sprintf("Wiederhole den Satz und achte besonders auf %q.", weakPhonemeWord)
case weakWord != "":
feedback = fmt.Sprintf("Das Wort %q wurde noch unsicher bewertet. Übe es im Satz noch einmal.", weakWord)
nextRecommendation = fmt.Sprintf("Wiederhole den Satz und achte besonders auf %q.", weakWord)
+28 -1
View File
@@ -39,6 +39,20 @@ func TestFeedbackForWeakWordNamesObservedProblem(t *testing.T) {
}
}
func TestFeedbackNamesMeasuredWeakPhoneme(t *testing.T) {
result := domain.PronunciationResult{
OverallScore: 79,
WordScores: []domain.WordScore{{
Word: "water", Accuracy: 72, Highlight: "weak",
PhonemeScores: []domain.PhonemeScore{{Phoneme: "w", Accuracy: 90}, {Phoneme: "t", Accuracy: 58}},
}},
}
feedback := mockFeedback("I drink water.", result)
if !strings.Contains(feedback.TeacherFeedback, "/t/") || !strings.Contains(feedback.TeacherFeedback, "58") {
t.Fatalf("expected measured phoneme in feedback, got %#v", feedback)
}
}
func TestFlowiseNextExerciseSendsSessionAndParsesResponse(t *testing.T) {
var requestBody map[string]json.RawMessage
@@ -78,7 +92,14 @@ func TestFlowiseNextExerciseSendsSessionAndParsesResponse(t *testing.T) {
}, slog.New(slog.NewTextHandler(io.Discard, nil)))
exercise, err := teacher.NextExercise(context.Background(), domain.GeneratedExerciseRequest{
UserID: "usr_1",
UserID: "usr_1",
Progress: domain.ProgressSummary{
WeakPhonemes: []domain.WeakPhoneme{{Word: "water", Phoneme: "t", Accuracy: 58}},
RecentAttempts: []domain.Attempt{{WordScores: []domain.WordScore{{
Word: "water", Accuracy: 70, PhonemeScores: []domain.PhonemeScore{{Phoneme: "t", Accuracy: 58}},
}}}},
},
RecentWeakWords: []domain.WordScore{{Word: "water", Accuracy: 70, PhonemeScores: []domain.PhonemeScore{{Phoneme: "t", Accuracy: 58}}}},
TargetLanguage: "en-US",
CEFRLevel: "A1",
SessionID: "usr_1:en-US:practice",
@@ -128,6 +149,12 @@ func TestFlowiseNextExerciseSendsSessionAndParsesResponse(t *testing.T) {
if texts, ok := question["avoid_reference_texts"].([]any); !ok || len(texts) != 2 {
t.Fatalf("unexpected avoid_reference_texts %#v", question["avoid_reference_texts"])
}
if phonemes, ok := question["recent_weak_phonemes"].([]any); !ok || len(phonemes) != 1 {
t.Fatalf("weak phonemes were not sent to Flowise: %#v", question["recent_weak_phonemes"])
}
if strings.Contains(questionText, "phoneme_scores") {
t.Fatal("raw phoneme scores should be summarized before sending progress to Flowise")
}
}
func TestValidGeneratedExerciseRejectsEarlierRecentSentence(t *testing.T) {
+22 -7
View File
@@ -157,7 +157,11 @@ type azurePronunciationResponse struct {
ErrorType string `json:"ErrorType"`
} `json:"PronunciationAssessment"`
Phonemes []struct {
Phoneme string `json:"Phoneme"`
Phoneme string `json:"Phoneme"`
AccuracyScore *float64 `json:"AccuracyScore"`
PronunciationAssessment struct {
AccuracyScore *float64 `json:"AccuracyScore"`
} `json:"PronunciationAssessment"`
} `json:"Phonemes"`
} `json:"Words"`
} `json:"NBest"`
@@ -183,18 +187,29 @@ func parseAzurePronunciation(body []byte) (domain.PronunciationResult, error) {
}
for _, word := range best.Words {
phonemes := make([]string, 0, len(word.Phonemes))
phonemeScores := make([]domain.PhonemeScore, 0, len(word.Phonemes))
for _, phoneme := range word.Phonemes {
if phoneme.Phoneme != "" {
phonemes = append(phonemes, phoneme.Phoneme)
accuracy := phoneme.PronunciationAssessment.AccuracyScore
if accuracy == nil {
accuracy = phoneme.AccuracyScore
}
if accuracy != nil {
phonemeScores = append(phonemeScores, domain.PhonemeScore{
Phoneme: phoneme.Phoneme, Accuracy: *accuracy, Highlight: highlightFor(*accuracy),
})
}
}
}
score := firstScore(word.AccuracyScore, word.PronunciationAssessment.AccuracyScore)
result.WordScores = append(result.WordScores, domain.WordScore{
Word: word.Word,
Accuracy: score,
ErrorType: firstNonEmpty(word.PronunciationAssessment.ErrorType, word.ErrorType),
Phonemes: phonemes,
Highlight: highlightFor(score),
Word: word.Word,
Accuracy: score,
ErrorType: firstNonEmpty(word.PronunciationAssessment.ErrorType, word.ErrorType),
Phonemes: phonemes,
PhonemeScores: phonemeScores,
Highlight: highlightFor(score),
})
}
return result, nil
@@ -238,7 +253,7 @@ func highlightFor(score float64) string {
switch {
case score >= 85:
return "good"
case score >= 70:
case score >= 75:
return "ok"
default:
return "weak"
+34
View File
@@ -57,4 +57,38 @@ func TestParseAzurePronunciationTopLevelScores(t *testing.T) {
if result.WordScores[0].Highlight != "good" {
t.Fatalf("expected good highlight, got %s", result.WordScores[0].Highlight)
}
if len(result.WordScores[0].PhonemeScores) != 0 {
t.Fatalf("unscored phonemes must not become zero-point weaknesses: %#v", result.WordScores[0].PhonemeScores)
}
}
func TestParseAzurePronunciationKeepsPhonemeScores(t *testing.T) {
body := []byte(`{
"NBest": [{"Display":"water","PronScore":82,"Words":[{
"Word":"water","AccuracyScore":80,
"Phonemes":[
{"Phoneme":"w","PronunciationAssessment":{"AccuracyScore":93}},
{"Phoneme":"ɔ","PronunciationAssessment":{"AccuracyScore":0}},
{"Phoneme":"t","AccuracyScore":58},
{"Phoneme":"ə"}
]
}]}]
}`)
result, err := parseAzurePronunciation(body)
if err != nil {
t.Fatalf("parse azure pronunciation: %v", err)
}
word := result.WordScores[0]
if len(word.Phonemes) != 4 || len(word.PhonemeScores) != 3 {
t.Fatalf("expected four phoneme labels and three scored phonemes, got %#v", word)
}
if word.PhonemeScores[0].Accuracy != 93 || word.PhonemeScores[0].Highlight != "good" {
t.Fatalf("unexpected first phoneme score %#v", word.PhonemeScores[0])
}
if word.PhonemeScores[1].Accuracy != 0 || word.PhonemeScores[1].Highlight != "weak" {
t.Fatalf("zero is a valid low score, got %#v", word.PhonemeScores[1])
}
if word.PhonemeScores[2].Accuracy != 58 || word.PhonemeScores[2].Highlight != "weak" {
t.Fatalf("top-level phoneme score was not read: %#v", word.PhonemeScores[2])
}
}
+53
View File
@@ -8,6 +8,7 @@ import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"speech-learning-api/internal/domain"
@@ -393,6 +394,7 @@ func (s *PostgresStore) Progress(ctx context.Context, userID string, limit int)
return domain.ProgressSummary{}, err
}
summary.WeakWords = latestWeakWords(summary.RecentAttempts)
summary.WeakPhonemes = latestWeakPhonemes(summary.RecentAttempts)
return summary, nil
}
@@ -416,6 +418,57 @@ func latestWeakWords(attempts []domain.Attempt) []domain.WordScore {
return weak
}
// Keep the newest scored occurrence for each word and phoneme. Within one
// attempt, use the lowest score if the same sound occurs more than once.
func latestWeakPhonemes(attempts []domain.Attempt) []domain.WeakPhoneme {
seen := make(map[string]bool)
var weak []domain.WeakPhoneme
for _, attempt := range attempts {
current := make(map[string]domain.WeakPhoneme)
for _, word := range attempt.WordScores {
if strings.EqualFold(word.ErrorType, "Omission") {
continue
}
wordKey := strings.ToLower(strings.TrimSpace(word.Word))
if wordKey == "" {
continue
}
for _, phoneme := range word.PhonemeScores {
name := strings.TrimSpace(phoneme.Phoneme)
if name == "" {
continue
}
key := wordKey + "\x00" + name
if seen[key] {
continue
}
if previous, ok := current[key]; !ok || phoneme.Accuracy < previous.Accuracy {
current[key] = domain.WeakPhoneme{Word: word.Word, Phoneme: name, Accuracy: phoneme.Accuracy}
}
}
}
for key, score := range current {
seen[key] = true
if score.Accuracy < 75 {
weak = append(weak, score)
}
}
}
sort.Slice(weak, func(i, j int) bool {
if weak[i].Accuracy == weak[j].Accuracy {
if weak[i].Word == weak[j].Word {
return weak[i].Phoneme < weak[j].Phoneme
}
return weak[i].Word < weak[j].Word
}
return weak[i].Accuracy < weak[j].Accuracy
})
if len(weak) > 12 {
weak = weak[:12]
}
return weak
}
func (s *PostgresStore) CreateConversationSession(ctx context.Context, userID, targetLanguage, cefrLevel string) (domain.ConversationSession, error) {
session := domain.ConversationSession{
ID: NewID("con"),
+17
View File
@@ -23,3 +23,20 @@ func TestLatestWeakWordsUsesNewestScorePerWord(t *testing.T) {
t.Fatalf("latest scores should clear water and keep coffee/please weak, got %#v", weak)
}
}
func TestLatestWeakPhonemesUsesNewestScoredAttempt(t *testing.T) {
attempts := []domain.Attempt{
{WordScores: []domain.WordScore{
{Word: "Water", PhonemeScores: []domain.PhonemeScore{{Phoneme: "w", Accuracy: 100}, {Phoneme: "t", Accuracy: 61}, {Phoneme: "t", Accuracy: 55}}},
}},
{WordScores: []domain.WordScore{
{Word: "water", PhonemeScores: []domain.PhonemeScore{{Phoneme: "w", Accuracy: 42}, {Phoneme: "t", Accuracy: 95}}},
{Word: "tea", PhonemeScores: []domain.PhonemeScore{{Phoneme: "iː", Accuracy: 68}}},
{Word: "missed", ErrorType: "Omission", PhonemeScores: []domain.PhonemeScore{{Phoneme: "m", Accuracy: 0}}},
}},
}
weak := latestWeakPhonemes(attempts)
if len(weak) != 2 || weak[0].Phoneme != "t" || weak[0].Accuracy != 55 || weak[1].Word != "tea" {
t.Fatalf("newer /w/ success should clear old weakness; latest /t/ and /iː/ should remain, got %#v", weak)
}
}
+14
View File
@@ -38,9 +38,22 @@ export type WordScore = {
accuracy: number;
error_type?: string;
phonemes?: string[];
phoneme_scores?: PhonemeScore[];
highlight: 'good' | 'ok' | 'weak';
};
export type PhonemeScore = {
phoneme: string;
accuracy: number;
highlight: 'good' | 'ok' | 'weak';
};
export type WeakPhoneme = {
word: string;
phoneme: string;
accuracy: number;
};
export type Attempt = {
id: string;
exercise_id: string;
@@ -60,6 +73,7 @@ export type ProgressSummary = {
total_attempts: number;
average_score: number;
weak_words: WordScore[] | null;
weak_phonemes: WeakPhoneme[] | null;
recent_attempts: Attempt[] | null;
daily_goal_minutes: number;
};
+16
View File
@@ -318,6 +318,22 @@ input:focus {
gap: 14px;
}
.phoneme-panel {
grid-column: 1 / -1;
}
.phoneme-row,
.phoneme-list {
display: flex;
align-items: center;
flex-wrap: wrap;
gap: 8px;
}
.phoneme-row strong {
min-width: 100px;
}
.debug-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
@@ -0,0 +1,38 @@
import type { WordScore } from '../api/types';
type Props = {
words: WordScore[];
};
export function PhonemeAnalysis({ words }: Props) {
const scoredWords = words.filter((word) => word.error_type?.toLowerCase() !== 'omission' && (word.phoneme_scores?.length ?? 0) > 0);
return (
<section className="panel result-panel phoneme-panel">
<h2>Lautanalyse</h2>
{scoredWords.length === 0 ? (
<p className="muted">Für diese Aufnahme sind keine Lautwerte verfügbar.</p>
) : (
<>
<p className="muted">Azure bewertet jeden Laut von 0 bis 100. Werte unter 75 markieren hier Übungsbedarf.</p>
{scoredWords.map((word, wordIndex) => (
<div className="phoneme-row" key={`${word.word}-${wordIndex}`}>
<strong>{word.word}</strong>
<div className="phoneme-list">
{word.phoneme_scores?.map((phoneme, phonemeIndex) => (
<span
className={`word word-${phoneme.highlight}`}
title={`${word.word}: Laut /${phoneme.phoneme}/, ${Math.round(phoneme.accuracy)} von 100 Punkten`}
key={`${phoneme.phoneme}-${phonemeIndex}`}
>
/{phoneme.phoneme}/ {Math.round(phoneme.accuracy)}
</span>
))}
</div>
</div>
))}
</>
)}
</section>
);
}
+2
View File
@@ -4,6 +4,7 @@ import { fetchNextLesson } from '../api/lessons';
import { submitAttempt } from '../api/practice';
import type { AttemptResponse, Exercise, Lesson } from '../api/types';
import { AudioRecorder } from '../components/AudioRecorder';
import { PhonemeAnalysis } from '../components/PhonemeAnalysis';
import { PronunciationText } from '../components/PronunciationText';
import { ScoreBadge } from '../components/ScoreBadge';
@@ -115,6 +116,7 @@ export function PracticePage({ initialSelection }: Props) {
<p>{result.teacher.teacher_feedback}</p>
<p className="muted">{result.teacher.next_recommendation}</p>
</div>
<PhonemeAnalysis words={result.attempt.word_scores} />
</section>
)}
</div>
+15
View File
@@ -48,6 +48,21 @@ export function ProgressPage() {
</div>
</section>
<section className="panel result-panel">
<h2>Schwierige Laute</h2>
<div className="weak-list">
{(progress.weak_phonemes ?? []).length === 0 ? (
<p className="muted">Noch keine bewerteten Lautschwächen.</p>
) : (
(progress.weak_phonemes ?? []).map((item) => (
<span className="word word-weak" key={`${item.word}-${item.phoneme}`}>
{item.word}: /{item.phoneme}/ {Math.round(item.accuracy)}
</span>
))
)}
</div>
</section>
<section className="panel result-panel">
<h2>Letzte Versuche</h2>
<div className="attempt-list">