package store import ( "context" "database/sql" "encoding/json" "errors" "fmt" "strings" "time" "speech-learning-api/internal/domain" "speech-learning-api/internal/pronunciation" ) const pronunciationSchema = ` ALTER TABLE attempts ADD COLUMN IF NOT EXISTS provider TEXT NOT NULL DEFAULT 'unknown'; ALTER TABLE attempts ADD COLUMN IF NOT EXISTS speech_metadata JSONB; CREATE TABLE IF NOT EXISTS pronunciation_assessments ( id TEXT PRIMARY KEY, user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, source TEXT NOT NULL,source_id TEXT NOT NULL,exercise_id TEXT NOT NULL, language TEXT NOT NULL,locale TEXT NOT NULL,alphabet TEXT NOT NULL,method TEXT NOT NULL, provider TEXT NOT NULL,measurement_version TEXT NOT NULL,data JSONB NOT NULL, exam_id TEXT REFERENCES course_v2_exams(id),created_at TIMESTAMPTZ NOT NULL, UNIQUE(source,source_id) ); CREATE INDEX IF NOT EXISTS pronunciation_assessment_history ON pronunciation_assessments(user_id,language,created_at DESC); CREATE TABLE IF NOT EXISTS pronunciation_observations ( assessment_id TEXT NOT NULL REFERENCES pronunciation_assessments(id) ON DELETE CASCADE, ordinal INT NOT NULL,kind TEXT NOT NULL,target_key TEXT NOT NULL,word TEXT NOT NULL, data JSONB NOT NULL,PRIMARY KEY(assessment_id,ordinal) ); CREATE INDEX IF NOT EXISTS pronunciation_observation_target ON pronunciation_observations(target_key,assessment_id); CREATE TABLE IF NOT EXISTS pronunciation_plans ( id TEXT PRIMARY KEY,user_id TEXT NOT NULL REFERENCES users(id) ON DELETE CASCADE, language TEXT NOT NULL,data JSONB NOT NULL,status TEXT NOT NULL CHECK(status IN ('pending','assigned','failed')), exercise_id TEXT UNIQUE REFERENCES exercises(id) ON DELETE SET NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now() ); CREATE INDEX IF NOT EXISTS pronunciation_plan_history ON pronunciation_plans(user_id,language,created_at DESC); ` func initPronunciation(ctx context.Context, db *sql.DB) error { _, err := db.ExecContext(ctx, pronunciationSchema) return err } // Metadata is recorded for all providers, but only genuine Azure scripted // measurements produce statistical observations. Missing locale names never // become guessed phoneme names. func measurementMetadata(r domain.PronunciationResult, language string) domain.PronunciationResult { if r.Locale == "" { r.Locale = language } if r.Method == "" { r.Method = "scripted" } if r.Alphabet == "" && r.Locale == "en-US" { r.Alphabet = "IPA" } if r.MeasurementVersion == "" { r.MeasurementVersion = "azure-rest-v1" } return r } func recordPronunciation(ctx context.Context, tx *sql.Tx, user, source, sourceID, exercise, language, exam string, at time.Time, result domain.PronunciationResult) error { r := measurementMetadata(result, language) id := source + ":" + sourceID data, err := json.Marshal(r) if err != nil { return err } res, err := tx.ExecContext(ctx, `INSERT INTO pronunciation_assessments(id,user_id,source,source_id,exercise_id,language,locale,alphabet,method,provider,measurement_version,data,exam_id,created_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,NULLIF($13,''),$14) ON CONFLICT(source,source_id) DO NOTHING`, id, user, source, sourceID, exercise, language, r.Locale, r.Alphabet, r.Method, r.Provider, r.MeasurementVersion, data, exam, at) if err != nil { return err } n, err := res.RowsAffected() if err != nil || n == 0 { return err } if r.Provider != "azure" || r.Method != "scripted" { return nil } observations := pronunciation.Observe(id, exercise, language, at, r) for i, o := range observations { raw, err := json.Marshal(o) if err != nil { return err } if _, err = tx.ExecContext(ctx, `INSERT INTO pronunciation_observations(assessment_id,ordinal,kind,target_key,word,data) VALUES($1,$2,$3,$4,$5,$6)`, id, i, o.Kind, pronunciation.Key(o.Kind, o.Locale, o.Alphabet, o.Symbol), o.Word, raw); err != nil { return err } } if source == "practice" { var raw []byte err := tx.QueryRowContext(ctx, `SELECT data FROM pronunciation_plans WHERE user_id=$1 AND exercise_id=$2 AND status='assigned' FOR UPDATE`, user, strings.TrimPrefix(exercise, "practice:")).Scan(&raw) if err == nil { var plan domain.PronunciationPlan if err = json.Unmarshal(raw, &plan); err != nil { return err } if plan.Mode == "transfer" && plan.Target != nil && plan.CandidateWord != "" { status := "unconfirmed" for _, o := range observations { if o.Kind == "phoneme" && o.Word == pronunciation.Word(plan.CandidateWord) { status = "measured_without_target" if o.Symbol == plan.Target.Symbol && o.Locale == plan.Target.Locale && o.Alphabet == plan.Target.Alphabet { status = "confirmed" break } } } plan.TransferStatus = status raw, err = json.Marshal(plan) if err != nil { return err } if _, err = tx.ExecContext(ctx, `UPDATE pronunciation_plans SET data=$2 WHERE id=$1`, plan.ID, raw); err != nil { return err } } } else if !errors.Is(err, sql.ErrNoRows) { return err } } return nil } // Confirmed historical course measurements have provenance. Legacy imports // and free speech do not enter the scripted profile. Historical practice rows // without metadata remain untouched and explicitly unverified. func (s *PostgresStore) BackfillPronunciation(ctx context.Context) error { rows, err := s.db.QueryContext(ctx, `SELECT a.id,a.user_id,COALESCE(NULLIF(t.data->>'evidence_task_id',''),a.task_id),t.language,COALESCE(a.exam_id,''),a.created_at,a.speech FROM course_v2_attempts a JOIN course_v2_tasks t ON t.id=a.task_id WHERE a.speech->>'provider'='azure' AND a.demo=false AND a.legacy_id IS NULL AND t.kind='repeat_sentence' AND NOT EXISTS(SELECT 1 FROM pronunciation_assessments p WHERE p.source='course' AND p.source_id=a.id)`) if err != nil { return err } type item struct { id, user, task, lang, exam string at time.Time raw []byte } items := []item{} for rows.Next() { var i item if err = rows.Scan(&i.id, &i.user, &i.task, &i.lang, &i.exam, &i.at, &i.raw); err != nil { rows.Close() return err } items = append(items, i) } err = rows.Err() rows.Close() if err != nil { return err } for _, i := range items { var r domain.PronunciationResult if err = json.Unmarshal(i.raw, &r); err != nil { return err } tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } err = recordPronunciation(ctx, tx, i.user, "course", i.id, "course:"+i.task, i.lang, i.exam, i.at, r) if err != nil { tx.Rollback() return err } if err = tx.Commit(); err != nil { return err } } return nil } type profileDB interface { QueryContext(context.Context, string, ...any) (*sql.Rows, error) QueryRowContext(context.Context, string, ...any) *sql.Row } func readPronunciationProfile(ctx context.Context, db profileDB, user, language string, now time.Time) (domain.PronunciationProfile, error) { rows, err := db.QueryContext(ctx, `SELECT o.data FROM pronunciation_observations o JOIN pronunciation_assessments a ON a.id=o.assessment_id LEFT JOIN course_v2_exams e ON e.id=a.exam_id WHERE a.user_id=$1 AND a.language=$2 AND a.provider='azure' AND a.method='scripted' AND a.created_at >= $3 AND a.created_at <= $4 AND (a.exam_id IS NULL OR e.status='completed')`, user, language, now.AddDate(0, 0, -90), now) if err != nil { return domain.PronunciationProfile{}, err } obs := []domain.PronunciationObservation{} for rows.Next() { var raw []byte var o domain.PronunciationObservation if err = rows.Scan(&raw); err != nil { rows.Close() return domain.PronunciationProfile{}, err } if err = json.Unmarshal(raw, &o); err != nil { rows.Close() return domain.PronunciationProfile{}, err } obs = append(obs, o) } err = rows.Err() rows.Close() if err != nil { return domain.PronunciationProfile{}, err } p := pronunciation.Build(language, now, obs) err = db.QueryRowContext(ctx, `SELECT count(*) FROM attempts WHERE user_id=$1 AND target_language=$2 AND provider='unknown'`, user, language).Scan(&p.UnverifiedHistory) return p, err } func (s *PostgresStore) PronunciationProfile(ctx context.Context, user, language string, now time.Time) (domain.PronunciationProfile, error) { return readPronunciationProfile(ctx, s.db, user, language, now) } func (s *PostgresStore) PlanPronunciation(ctx context.Context, user, language string, now time.Time) (domain.PronunciationPlan, error) { tx, err := s.db.BeginTx(ctx, nil) if err != nil { return domain.PronunciationPlan{}, err } defer tx.Rollback() if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, "pronunciation:"+user+":"+language); err != nil { return domain.PronunciationPlan{}, err } // Abandoned reservations never permanently distort the learning mix. if _, err = tx.ExecContext(ctx, `UPDATE pronunciation_plans SET status='failed' WHERE user_id=$1 AND language=$2 AND status='pending' AND created_at<$3`, user, language, now.Add(-5*time.Minute)); err != nil { return domain.PronunciationPlan{}, err } p, err := readPronunciationProfile(ctx, tx, user, language, now) if err != nil { return domain.PronunciationPlan{}, err } rows, err := tx.QueryContext(ctx, `SELECT jsonb_build_object('category',data->'category','target',CASE WHEN data->'target' IS NOT NULL THEN jsonb_build_object('key',data->'target'->'key') ELSE NULL END) FROM pronunciation_plans WHERE user_id=$1 AND language=$2 AND status IN ('assigned','pending') ORDER BY created_at DESC,id DESC`, user, language) if err != nil { return domain.PronunciationPlan{}, err } history := []domain.PronunciationPlan{} for rows.Next() { var raw []byte var h domain.PronunciationPlan if err = rows.Scan(&raw); err != nil { rows.Close() return domain.PronunciationPlan{}, err } if err = json.Unmarshal(raw, &h); err != nil { rows.Close() return domain.PronunciationPlan{}, err } history = append(history, h) } err = rows.Err() rows.Close() if err != nil { return domain.PronunciationPlan{}, err } plan := pronunciation.Select(p, history, now) plan.ID = NewID("pronplan") raw, err := json.Marshal(plan) if err != nil { return plan, err } _, err = tx.ExecContext(ctx, `INSERT INTO pronunciation_plans(id,user_id,language,data,status,created_at) VALUES($1,$2,$3,$4,'pending',$5)`, plan.ID, user, language, raw, now) if err != nil { return plan, err } return plan, tx.Commit() } func (s *PostgresStore) FailPronunciationPlan(ctx context.Context, user, id string) error { _, err := s.db.ExecContext(ctx, `UPDATE pronunciation_plans SET status='failed' WHERE id=$1 AND user_id=$2 AND status='pending'`, id, user) return err } func savePronunciationPlan(ctx context.Context, tx *sql.Tx, user, exercise string, plan *domain.PronunciationPlan) error { if plan == nil { return nil } raw, err := json.Marshal(plan) if err != nil { return err } res, err := tx.ExecContext(ctx, `UPDATE pronunciation_plans SET status='assigned',exercise_id=$3,data=$4 WHERE id=$1 AND user_id=$2 AND language=$5 AND status='pending'`, plan.ID, user, exercise, raw, plan.Language) if err != nil { return err } n, err := res.RowsAffected() if err != nil { return err } if n != 1 { return fmt.Errorf("pronunciation plan is no longer assignable") } return nil }