Files
Speech/apps/api/internal/store/course_bank.go
T
Eugenius Naumenko 16cceae28a Muttersprache koppeln und russischen A1-Kurs ergänzen
GUI und neue Erklärungen verwenden gemeinsam native_language. Deutsch und
Russisch sind die beiden Muttersprachen; Englisch bleibt die Lernsprache.
Kompatibilitätsfelder werden atomar gekoppelt, widersprüchliche Eingaben
abgewiesen und bestehende Profile sowie eingefrorene Inhalte erhalten.

Ergänzt vollständige russische GUI-Kataloge und 840 geprüfte Aufgabenansichten
in sieben Modulen einschließlich A1-Test, mit getrennten Sprachrollen für
Ausgangs- und Lernmaterial. Äquivalente Ansichten teilen ihre Nachweisidentität.
Stabile API-Fehlercodes, neue deterministische Hilfen und Trainingsbegründungen
berücksichtigen die Muttersprache. Dokumentiert das Sprachmodell in AGENTS.md.

Validierung: Go-Race-Suite mit eigener lokaler PostgreSQL-Testdatenbank, go vet,
Web-Build, Offline-Bank-/Variantenaudit, Diff-Prüfung sowie lokale Browser-Smokes
für Sprachen, Kurs, Administration, Aussprache und Aufnahme. Keine Live-Anbieter,
Produktionsprofile, Deployment-Schritte oder historischen Neubewertungen.
2026-11-07 17:02:17 +00:00

501 lines
20 KiB
Go

package store
import (
"context"
"crypto/sha256"
"database/sql"
"embed"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io/fs"
"sort"
"speech-learning-api/internal/domain"
"strings"
)
//go:embed coursebank/en-US/a1-v2/*.json
var bankFiles embed.FS
const bankSchema = `
CREATE TABLE IF NOT EXISTS course_v2_bank (
id TEXT PRIMARY KEY, language TEXT NOT NULL, version TEXT NOT NULL, revision TEXT NOT NULL,
module_id TEXT NOT NULL, skill TEXT NOT NULL, goal TEXT NOT NULL, difficulty INT NOT NULL CHECK(difficulty BETWEEN 1 AND 3),
data JSONB NOT NULL, checksum TEXT NOT NULL, active BOOLEAN NOT NULL DEFAULT true,
FOREIGN KEY(module_id,language,version) REFERENCES course_v2_modules(id,language,version));
CREATE INDEX IF NOT EXISTS course_v2_bank_lookup ON course_v2_bank(language,version,module_id,skill) WHERE active;
ALTER TABLE course_v2_tasks ADD COLUMN IF NOT EXISTS bank_entry_id TEXT REFERENCES course_v2_bank(id);
ALTER TABLE course_v2_tasks ADD COLUMN IF NOT EXISTS last_presented_at TIMESTAMPTZ;
ALTER TABLE course_v2_tasks ADD COLUMN IF NOT EXISTS presentation_count INT NOT NULL DEFAULT 0;
UPDATE course_v2_tasks SET presentation_count=1 WHERE source='curated' AND bank_entry_id IS NOT NULL AND presentation_count=0;
ALTER TABLE course_v2_tasks DROP CONSTRAINT IF EXISTS course_v2_tasks_source_check;
ALTER TABLE course_v2_tasks ADD CONSTRAINT course_v2_tasks_source_check CHECK(source IN ('seed','flowise','ai','curated'));
ALTER TABLE course_v2_attempts DROP CONSTRAINT IF EXISTS course_v2_attempts_status_check;
ALTER TABLE course_v2_attempts ADD CONSTRAINT course_v2_attempts_status_check CHECK(status IN ('pending','graded','failed','ungraded'));
`
func BundledBank() ([]domain.BankEntry, error) {
paths, err := fs.Glob(bankFiles, "coursebank/en-US/a1-v2/*.json")
if err != nil {
return nil, err
}
var manifest struct {
Total int `json:"total"`
Files map[string]string `json:"files"`
}
manifestData, err := bankFiles.ReadFile("coursebank/en-US/a1-v2/manifest.json")
if err != nil {
return nil, err
}
if err = json.Unmarshal(manifestData, &manifest); err != nil {
return nil, err
}
if manifest.Total != 840 || len(manifest.Files) != 42 {
return nil, errors.New("invalid bank manifest")
}
var entries []domain.BankEntry
for _, p := range paths {
if strings.HasSuffix(p, "/manifest.json") {
continue
}
data, err := bankFiles.ReadFile(p)
if err != nil {
return nil, err
}
sum := sha256.Sum256(data)
name := strings.TrimPrefix(p, "coursebank/en-US/a1-v2/")
if manifest.Files[name] != hex.EncodeToString(sum[:]) {
return nil, fmt.Errorf("bank file checksum mismatch: %s", name)
}
var batch []domain.BankEntry
dec := json.NewDecoder(strings.NewReader(string(data)))
dec.DisallowUnknownFields()
if err = dec.Decode(&batch); err != nil {
return nil, fmt.Errorf("bank %s: %w", p, err)
}
entries = append(entries, batch...)
}
return entries, ValidateBank(entries)
}
func ValidateBank(entries []domain.BankEntry) error {
if len(entries) != 840 {
return fmt.Errorf("bank requires 840 entries, got %d", len(entries))
}
modules := map[string]bool{}
for _, m := range EnglishCoursePack().Modules {
modules[m.ID] = true
}
seen := map[string]bool{}
fingerprints := map[string]bool{}
coverage := map[string]int{}
levels := map[string][3]int{}
goals := map[string]map[string]bool{}
for _, e := range entries {
t := e.Task
k := t.ModuleID + ":" + t.Skill
if e.ID == "" || seen[e.ID] || e.Revision == "" || !e.Reviewed || e.Goal == "" || e.Difficulty < 1 || e.Difficulty > 3 || !modules[t.ModuleID] || t.TargetLanguage != "en-US" || t.CourseVersion != domain.CourseVersion || t.SchemaVersion != 2 || t.CEFRLevel != "A1" || t.AnswerKey == nil || t.ID != "" || t.Source != "" || t.ExamID != "" {
return fmt.Errorf("invalid bank entry %s", e.ID)
}
if len([]rune(t.Prompt)) > 2000 || len([]rune(t.Passage)) > 10000 || len([]rune(t.AudioText)) > 4000 || len([]rune(t.ReferenceText)) > 500 || strings.Contains(t.Prompt, "<") || t.Prompt == "" || len(t.AnswerKey.Rubric) == 0 || t.AnswerKey.Explanation == "" {
return fmt.Errorf("incomplete bank entry %s", e.ID)
}
if t.Skill == "reading" && t.Passage == "" || t.Skill == "listening" && t.AudioText == "" || t.Kind == "repeat_sentence" && t.ReferenceText == "" {
return fmt.Errorf("missing media %s", e.ID)
}
if !bankMember(t.Skill, domain.CourseSkills) || !bankMember(t.Kind, []string{"choice", "short_text", "free_text", "repeat_sentence", "free_speech"}) {
return fmt.Errorf("invalid renderer %s", e.ID)
}
if t.Skill == "writing" && !bankMember(t.Kind, []string{"short_text", "free_text"}) || t.Skill == "speaking" && !bankMember(t.Kind, []string{"repeat_sentence", "free_speech"}) {
return fmt.Errorf("invalid skill renderer %s", e.ID)
}
if t.Kind == "choice" {
if len(t.Options) != 3 {
return fmt.Errorf("three options required %s", e.ID)
}
ids := map[string]bool{}
labels := map[string]bool{}
found := false
for _, o := range t.Options {
label := strings.ToLower(strings.TrimSpace(o.Label))
if o.ID == "" || label == "" || ids[o.ID] || labels[label] {
return fmt.Errorf("duplicate option %s", e.ID)
}
ids[o.ID] = true
labels[label] = true
found = found || o.ID == t.AnswerKey.CorrectOptionID
}
if !found {
return fmt.Errorf("invalid solution %s", e.ID)
}
} else if len(t.Options) != 0 {
return fmt.Errorf("unexpected options %s", e.ID)
}
fp := fingerprint(t)
if fingerprints[fp] {
return fmt.Errorf("duplicate bank content %s", e.ID)
}
fingerprints[fp] = true
seen[e.ID] = true
coverage[k]++
v := levels[k]
v[e.Difficulty-1]++
levels[k] = v
if goals[k] == nil {
goals[k] = map[string]bool{}
}
goals[k][e.Goal] = true
}
for m := range modules {
for _, skill := range domain.CourseSkills {
k := m + ":" + skill
if coverage[k] != 20 || levels[k] != [3]int{8, 8, 4} || len(goals[k]) < 5 {
return fmt.Errorf("invalid bank coverage %s", k)
}
}
}
return nil
}
func bankMember(value string, list []string) bool {
for _, s := range list {
if s == value {
return true
}
}
return false
}
func (s *AdaptiveStore) InstallBank(ctx context.Context) error {
entries, err := BundledBank()
if err != nil {
return err
}
if _, err = s.db.ExecContext(ctx, bankSchema); err != nil {
return err
}
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer tx.Rollback()
// Retire only templates. Assigned tasks and their frozen keys are never edited.
if _, err = tx.ExecContext(ctx, `UPDATE course_v2_bank SET active=false WHERE language='en-US' AND version=$1`, domain.CourseVersion); err != nil {
return err
}
for _, e := range entries {
data, _ := json.Marshal(e)
sum := sha256.Sum256(data)
checksum := hex.EncodeToString(sum[:])
var old string
err = tx.QueryRowContext(ctx, `SELECT checksum FROM course_v2_bank WHERE id=$1`, e.ID).Scan(&old)
if err == nil && old != checksum {
return fmt.Errorf("bank revision is immutable: %s", e.ID)
}
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return err
}
_, err = tx.ExecContext(ctx, `INSERT INTO course_v2_bank(id,language,version,revision,module_id,skill,goal,difficulty,data,checksum) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT(id) DO UPDATE SET active=true`, e.ID, e.Task.TargetLanguage, e.Task.CourseVersion, e.Revision, e.Task.ModuleID, e.Task.Skill, e.Goal, e.Difficulty, data, checksum)
if err != nil {
return err
}
}
if err = tx.Commit(); err != nil {
return err
}
return s.installVariants(ctx, entries)
}
type bankCandidate struct {
entry domain.BankEntry
bankEntryID string
variantID string
previousID string
taskID string
score float64
seen bool
presented int64
}
// AssignBankTask serializes selections per learner, including presentation of
// unanswered tasks. Merely clicking Next is not a graded learning result.
func (s *AdaptiveStore) AssignBankTask(ctx context.Context, j domain.CourseJob) (domain.AdaptiveTask, error) {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return domain.AdaptiveTask{}, err
}
defer tx.Rollback()
if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, j.UserID+":"+j.Language+":"+domain.CourseVersion); err != nil {
return domain.AdaptiveTask{}, err
}
var result []byte
err = tx.QueryRowContext(ctx, `SELECT result FROM course_v2_jobs WHERE id=$1 AND user_id=$2 AND lease=$3 AND status='running' FOR UPDATE`, j.ID, j.UserID, j.Lease).Scan(&result)
if err != nil {
return domain.AdaptiveTask{}, err
}
if len(result) > 0 {
var old struct {
TaskID string `json:"task_id"`
Review bool `json:"review"`
}
if err = json.Unmarshal(result, &old); err != nil {
return domain.AdaptiveTask{}, err
}
if old.TaskID != "" {
if err = tx.Commit(); err != nil {
return domain.AdaptiveTask{}, err
}
t, err := s.Task(ctx, j.UserID, j.Language, old.TaskID)
t.Review = old.Review
return t, err
}
}
instructionLanguage := valueOr(j.InstructionLanguage, "de-DE")
rows, err := tx.QueryContext(ctx, availableBank+`
SELECT av.entry,COALESCE(t.id,''),COALESCE(graded.score,-1),COALESCE(extract(epoch from previous.last_presented_at)::bigint,0),av.base_id,av.variant_id,COALESCE(previous.id,'')
FROM available av
LEFT JOIN course_v2_tasks t ON t.bank_entry_id=av.base_id AND COALESCE(t.variant_id,'')=av.variant_id AND t.owner_id=$1 AND t.exam_id IS NULL
LEFT JOIN LATERAL (SELECT r.id,r.last_presented_at FROM course_v2_tasks r WHERE r.owner_id=$1 AND r.language=$2 AND r.version=$3 AND r.exam_id IS NULL AND COALESCE(NULLIF(r.data->>'canonical_task_id',''),r.bank_entry_id)=av.canonical_id ORDER BY r.last_presented_at DESC NULLS LAST,r.id LIMIT 1) previous ON true
LEFT JOIN LATERAL (SELECT (a.evaluation->>'score')::float8 AS score FROM course_v2_attempts a JOIN course_v2_tasks r ON r.id=a.task_id WHERE a.user_id=$1 AND r.owner_id=$1 AND r.language=$2 AND r.version=$3 AND r.exam_id IS NULL AND COALESCE(NULLIF(r.data->>'canonical_task_id',''),r.bank_entry_id)=av.canonical_id AND a.status='graded' AND NOT a.demo ORDER BY a.created_at DESC,a.id DESC LIMIT 1) graded ON true
WHERE av.module_id=$5 AND ($6='' OR av.skill=$6) AND ($7=false OR av.entry->'task'->>'kind'='choice')`, j.UserID, j.Language, domain.CourseVersion, instructionLanguage, j.ModuleID, j.Skill, j.Kind == "legacy_next")
if err != nil {
return domain.AdaptiveTask{}, err
}
candidates := []bankCandidate{}
for rows.Next() {
var c bankCandidate
var data []byte
if err = rows.Scan(&data, &c.taskID, &c.score, &c.presented, &c.bankEntryID, &c.variantID, &c.previousID); err != nil {
rows.Close()
return domain.AdaptiveTask{}, err
}
if err = json.Unmarshal(data, &c.entry); err != nil {
rows.Close()
return domain.AdaptiveTask{}, err
}
c.seen = c.previousID != ""
candidates = append(candidates, c)
}
err = rows.Err()
rows.Close()
if err != nil {
return domain.AdaptiveTask{}, err
}
if len(candidates) == 0 {
if instructionLanguage != "de-DE" {
return domain.AdaptiveTask{}, ErrCourseVariantUnavailable
}
return domain.AdaptiveTask{}, ErrNotFound
}
type recent struct {
goal string
score float64
task string
difficulty int
}
recentScores := []recent{}
rows, err = tx.QueryContext(ctx, `SELECT b.goal,(a.evaluation->>'score')::float8,COALESCE(NULLIF(t.data->>'evidence_task_id',''),t.id),b.difficulty FROM course_v2_attempts a JOIN course_v2_tasks t ON t.id=a.task_id JOIN course_v2_bank b ON b.id=t.bank_entry_id WHERE a.user_id=$1 AND t.language=$2 AND t.version=$3 AND t.module_id=$4 AND ($5='' OR t.skill=$5) AND a.status='graded' AND NOT a.demo AND a.exam_id IS NULL ORDER BY a.created_at DESC,a.id DESC LIMIT 5`, j.UserID, j.Language, domain.CourseVersion, j.ModuleID, j.Skill)
if err != nil {
return domain.AdaptiveTask{}, err
}
for rows.Next() {
var r recent
if err = rows.Scan(&r.goal, &r.score, &r.task, &r.difficulty); err != nil {
rows.Close()
return domain.AdaptiveTask{}, err
}
recentScores = append(recentScores, r)
}
err = rows.Err()
rows.Close()
if err != nil {
return domain.AdaptiveTask{}, err
}
target := 1
weak := map[string]bool{}
distinct := map[string]bool{}
sum := float64(0)
for _, r := range recentScores {
sum += r.score
distinct[r.task] = true
if r.score < 80 {
weak[r.goal] = true
}
}
if len(recentScores) > 0 {
avg := sum / float64(len(recentScores))
target = recentScores[0].difficulty
if len(distinct) >= 3 && avg >= 85 && target < 3 {
target++
} else if len(distinct) >= 2 && avg < 60 && target > 1 {
target--
}
}
var lastID string
_ = tx.QueryRowContext(ctx, `SELECT id FROM course_v2_tasks WHERE owner_id=$1 AND language=$2 AND version=$3 AND module_id=$4 AND ($5='' OR skill=$5) AND exam_id IS NULL AND bank_entry_id IS NOT NULL ORDER BY last_presented_at DESC,id LIMIT 1`, j.UserID, j.Language, domain.CourseVersion, j.ModuleID, j.Skill).Scan(&lastID)
rank := func(c bankCandidate) int {
if !c.seen {
if weak[c.entry.Goal] && c.entry.Difficulty <= target {
return 0
}
return 1
}
if c.score >= 0 && c.score < 80 {
return 2
}
return 3
}
sort.Slice(candidates, func(i, k int) bool {
a, b := candidates[i], candidates[k]
if a.seen && b.seen && (a.previousID == lastID) != (b.previousID == lastID) {
return b.previousID == lastID
}
if rank(a) != rank(b) {
return rank(a) < rank(b)
}
if a.seen && b.seen {
if (a.previousID == lastID) != (b.previousID == lastID) {
return b.previousID == lastID
}
if a.presented != b.presented {
return a.presented < b.presented
}
}
da, db := a.entry.Difficulty-target, b.entry.Difficulty-target
if da < 0 {
da = -da
}
if db < 0 {
db = -db
}
if da != db {
return da < db
}
ha := sha256.Sum256([]byte(j.UserID + a.entry.ID))
hb := sha256.Sum256([]byte(j.UserID + b.entry.ID))
return strings.Compare(hex.EncodeToString(ha[:]), hex.EncodeToString(hb[:])) < 0
})
chosen := candidates[0]
t := chosen.entry.Task
t.Source = "curated"
t.Review = chosen.seen
t.ID = chosen.taskID
if chosen.taskID == "" {
t.ID = NewID("a2")
if t.InstructionLanguage == "" {
t.InstructionLanguage = "de-DE"
}
if t.CanonicalTaskID == "" {
t.CanonicalTaskID = chosen.bankEntryID
}
// Reuse the earliest evidence identity without editing any historic snapshot.
err = tx.QueryRowContext(ctx, `SELECT COALESCE(NULLIF(data->>'evidence_task_id',''),id) FROM course_v2_tasks WHERE owner_id=$1 AND language=$2 AND version=$3 AND COALESCE(NULLIF(data->>'canonical_task_id',''),bank_entry_id)=$4 ORDER BY created_at,id LIMIT 1`, j.UserID, j.Language, domain.CourseVersion, t.CanonicalTaskID).Scan(&t.EvidenceTaskID)
if errors.Is(err, sql.ErrNoRows) {
t.EvidenceTaskID = t.ID
} else if err != nil {
return t, err
}
data, _ := json.Marshal(t)
_, err = tx.ExecContext(ctx, `INSERT INTO course_v2_tasks(id,module_id,language,version,owner_id,source,skill,kind,data,fingerprint,bank_entry_id,variant_id,last_presented_at,presentation_count) VALUES($1,$2,$3,$4,$5,'curated',$6,$7,$8,$9,$10,NULLIF($11,''),now(),1)`, t.ID, t.ModuleID, t.TargetLanguage, t.CourseVersion, j.UserID, t.Skill, t.Kind, data, "bank:"+chosen.entry.ID, chosen.bankEntryID, chosen.variantID)
} else {
_, err = tx.ExecContext(ctx, `UPDATE course_v2_tasks SET last_presented_at=now(),presentation_count=presentation_count+1 WHERE id=$1`, t.ID)
}
if err != nil {
return domain.AdaptiveTask{}, err
}
data, _ := json.Marshal(map[string]any{"task_id": t.ID, "source": "curated", "review": chosen.seen})
if _, err = tx.ExecContext(ctx, `UPDATE course_v2_jobs SET result=$2 WHERE id=$1`, j.ID, data); err != nil {
return domain.AdaptiveTask{}, err
}
if err = tx.Commit(); err != nil {
return domain.AdaptiveTask{}, err
}
out, err := s.Task(ctx, j.UserID, j.Language, t.ID)
out.Review = chosen.seen
return out, err
}
func (s *AdaptiveStore) BankExam(ctx context.Context, user, language, module string, native ...string) ([]domain.AdaptiveTask, error) {
out := []domain.AdaptiveTask{}
mother := "de-DE"
if len(native) > 0 && native[0] != "" {
mother = native[0]
}
mods, err := s.Modules(ctx, language)
if err != nil {
return nil, err
}
if len(mods) != 7 {
return nil, ErrNotFound
}
for i, skill := range domain.CourseSkills {
for n := 0; n < 2; n++ {
origin := mods[(i+n+1)%6].Key
var data []byte
err = s.db.QueryRowContext(ctx, availableBank+` SELECT av.entry FROM available av WHERE av.module_id=$5 AND av.skill=$6 AND av.entry->'task'->>'origin_module_key'=$7 ORDER BY (SELECT count(*) FROM course_v2_tasks t WHERE t.owner_id=$1 AND (t.bank_entry_id=av.base_id OR t.fingerprint='bank-exam:'||(CASE WHEN av.variant_id='' THEN av.base_id ELSE av.variant_id END))),md5((av.entry->>'id')||$1) LIMIT 1`, user, language, domain.CourseVersion, mother, module, skill, origin).Scan(&data)
if errors.Is(err, sql.ErrNoRows) {
return nil, ErrNotFound
}
if err != nil {
return nil, err
}
var entry domain.BankEntry
if err = json.Unmarshal(data, &entry); err != nil {
return nil, err
}
t := entry.Task
t.Source = "curated"
t.BriefID = entry.ID
if t.CanonicalTaskID == "" {
t.CanonicalTaskID = entry.ID
}
if t.InstructionLanguage == "" {
t.InstructionLanguage = "de-DE"
}
out = append(out, t)
}
}
return out, nil
}
func (s *AdaptiveStore) Ungrade(ctx context.Context, user, id string) error {
_, err := s.db.ExecContext(ctx, `UPDATE course_v2_attempts SET status='ungraded' WHERE id=$1 AND user_id=$2 AND status='pending'`, id, user)
return err
}
type BankInventoryItem struct {
ModuleID string `json:"module_id"`
Skill string `json:"skill"`
Total int `json:"total"`
Unseen int `json:"unseen"`
}
func (s *AdaptiveStore) BankInventory(ctx context.Context, user, language string) ([]BankInventoryItem, error) {
var instructionLanguage string
if err := s.db.QueryRowContext(ctx, `SELECT instruction_language FROM users WHERE id=$1`, user).Scan(&instructionLanguage); err != nil {
return nil, err
}
rows, err := s.db.QueryContext(ctx, availableBank+` SELECT av.module_id,av.skill,count(*),count(*) FILTER(WHERE NOT EXISTS(SELECT 1 FROM course_v2_tasks t WHERE t.owner_id=$1 AND t.language=$2 AND t.version=$3 AND t.exam_id IS NULL AND COALESCE(NULLIF(t.data->>'canonical_task_id',''),t.bank_entry_id)=av.canonical_id)) FROM available av GROUP BY av.module_id,av.skill ORDER BY av.module_id,av.skill`, user, language, domain.CourseVersion, instructionLanguage)
if err != nil {
return nil, err
}
defer rows.Close()
out := []BankInventoryItem{}
for rows.Next() {
var item BankInventoryItem
if err = rows.Scan(&item.ModuleID, &item.Skill, &item.Total, &item.Unseen); err != nil {
return nil, err
}
out = append(out, item)
}
return out, rows.Err()
}
// Also repairs a crash between completing scores and storing the static summary.
// Existing ready/historical summaries are not overwritten.
func (s *AdaptiveStore) EnsureBankExamSummary(ctx context.Context, user, id string) error {
_, err := s.db.ExecContext(ctx, `UPDATE course_v2_exams e SET result=result||jsonb_build_object('summary',CASE WHEN (SELECT t.data->>'instruction_language' FROM course_v2_tasks t WHERE t.owner_id=$2 AND t.exam_id=e.id ORDER BY t.id LIMIT 1)='ru-RU' THEN $3::text ELSE $4::text END,'summary_status','ready') WHERE e.id=$1 AND e.user_id=$2 AND e.status='completed' AND e.result->>'summary_status'='pending'`, id, user,
"Внутренняя оценка уровня A1 на основе сохранённых результатов. Это не сертификат.",
"Interne A1-Lerneinschätzung aus den gespeicherten Bewertungen. Kein Zertifikat.")
return err
}