package store import ( "context" "crypto/sha256" "database/sql" "embed" "encoding/hex" "encoding/json" "errors" "fmt" "io/fs" "sort" "speech-learning-api/internal/domain" "strings" ) //go:embed coursebank/en-US/a1-v2/*.json var bankFiles embed.FS const bankSchema = ` CREATE TABLE IF NOT EXISTS course_v2_bank ( id TEXT PRIMARY KEY, language TEXT NOT NULL, version TEXT NOT NULL, revision TEXT NOT NULL, module_id TEXT NOT NULL, skill TEXT NOT NULL, goal TEXT NOT NULL, difficulty INT NOT NULL CHECK(difficulty BETWEEN 1 AND 3), data JSONB NOT NULL, checksum TEXT NOT NULL, active BOOLEAN NOT NULL DEFAULT true, FOREIGN KEY(module_id,language,version) REFERENCES course_v2_modules(id,language,version)); CREATE INDEX IF NOT EXISTS course_v2_bank_lookup ON course_v2_bank(language,version,module_id,skill) WHERE active; ALTER TABLE course_v2_tasks ADD COLUMN IF NOT EXISTS bank_entry_id TEXT REFERENCES course_v2_bank(id); ALTER TABLE course_v2_tasks ADD COLUMN IF NOT EXISTS last_presented_at TIMESTAMPTZ; ALTER TABLE course_v2_tasks ADD COLUMN IF NOT EXISTS presentation_count INT NOT NULL DEFAULT 0; UPDATE course_v2_tasks SET presentation_count=1 WHERE source='curated' AND bank_entry_id IS NOT NULL AND presentation_count=0; ALTER TABLE course_v2_tasks DROP CONSTRAINT IF EXISTS course_v2_tasks_source_check; ALTER TABLE course_v2_tasks ADD CONSTRAINT course_v2_tasks_source_check CHECK(source IN ('seed','flowise','ai','curated')); ALTER TABLE course_v2_attempts DROP CONSTRAINT IF EXISTS course_v2_attempts_status_check; ALTER TABLE course_v2_attempts ADD CONSTRAINT course_v2_attempts_status_check CHECK(status IN ('pending','graded','failed','ungraded')); ` func BundledBank() ([]domain.BankEntry, error) { paths, err := fs.Glob(bankFiles, "coursebank/en-US/a1-v2/*.json") if err != nil { return nil, err } var manifest struct { Total int `json:"total"` Files map[string]string `json:"files"` } manifestData, err := bankFiles.ReadFile("coursebank/en-US/a1-v2/manifest.json") if err != nil { return nil, err } if err = json.Unmarshal(manifestData, &manifest); err != nil { return nil, err } if manifest.Total != 840 || len(manifest.Files) != 42 { return nil, errors.New("invalid bank manifest") } var entries []domain.BankEntry for _, p := range paths { if strings.HasSuffix(p, "/manifest.json") { continue } data, err := bankFiles.ReadFile(p) if err != nil { return nil, err } sum := sha256.Sum256(data) name := strings.TrimPrefix(p, "coursebank/en-US/a1-v2/") if manifest.Files[name] != hex.EncodeToString(sum[:]) { return nil, fmt.Errorf("bank file checksum mismatch: %s", name) } var batch []domain.BankEntry dec := json.NewDecoder(strings.NewReader(string(data))) dec.DisallowUnknownFields() if err = dec.Decode(&batch); err != nil { return nil, fmt.Errorf("bank %s: %w", p, err) } entries = append(entries, batch...) } return entries, ValidateBank(entries) } func ValidateBank(entries []domain.BankEntry) error { if len(entries) != 840 { return fmt.Errorf("bank requires 840 entries, got %d", len(entries)) } modules := map[string]bool{} for _, m := range EnglishCoursePack().Modules { modules[m.ID] = true } seen := map[string]bool{} fingerprints := map[string]bool{} coverage := map[string]int{} levels := map[string][3]int{} goals := map[string]map[string]bool{} for _, e := range entries { t := e.Task k := t.ModuleID + ":" + t.Skill if e.ID == "" || seen[e.ID] || e.Revision == "" || !e.Reviewed || e.Goal == "" || e.Difficulty < 1 || e.Difficulty > 3 || !modules[t.ModuleID] || t.TargetLanguage != "en-US" || t.CourseVersion != domain.CourseVersion || t.SchemaVersion != 2 || t.CEFRLevel != "A1" || t.AnswerKey == nil || t.ID != "" || t.Source != "" || t.ExamID != "" { return fmt.Errorf("invalid bank entry %s", e.ID) } if len([]rune(t.Prompt)) > 2000 || len([]rune(t.Passage)) > 10000 || len([]rune(t.AudioText)) > 4000 || len([]rune(t.ReferenceText)) > 500 || strings.Contains(t.Prompt, "<") || t.Prompt == "" || len(t.AnswerKey.Rubric) == 0 || t.AnswerKey.Explanation == "" { return fmt.Errorf("incomplete bank entry %s", e.ID) } if t.Skill == "reading" && t.Passage == "" || t.Skill == "listening" && t.AudioText == "" || t.Kind == "repeat_sentence" && t.ReferenceText == "" { return fmt.Errorf("missing media %s", e.ID) } if !bankMember(t.Skill, domain.CourseSkills) || !bankMember(t.Kind, []string{"choice", "short_text", "free_text", "repeat_sentence", "free_speech"}) { return fmt.Errorf("invalid renderer %s", e.ID) } if t.Skill == "writing" && !bankMember(t.Kind, []string{"short_text", "free_text"}) || t.Skill == "speaking" && !bankMember(t.Kind, []string{"repeat_sentence", "free_speech"}) { return fmt.Errorf("invalid skill renderer %s", e.ID) } if t.Kind == "choice" { if len(t.Options) != 3 { return fmt.Errorf("three options required %s", e.ID) } ids := map[string]bool{} labels := map[string]bool{} found := false for _, o := range t.Options { label := strings.ToLower(strings.TrimSpace(o.Label)) if o.ID == "" || label == "" || ids[o.ID] || labels[label] { return fmt.Errorf("duplicate option %s", e.ID) } ids[o.ID] = true labels[label] = true found = found || o.ID == t.AnswerKey.CorrectOptionID } if !found { return fmt.Errorf("invalid solution %s", e.ID) } } else if len(t.Options) != 0 { return fmt.Errorf("unexpected options %s", e.ID) } fp := fingerprint(t) if fingerprints[fp] { return fmt.Errorf("duplicate bank content %s", e.ID) } fingerprints[fp] = true seen[e.ID] = true coverage[k]++ v := levels[k] v[e.Difficulty-1]++ levels[k] = v if goals[k] == nil { goals[k] = map[string]bool{} } goals[k][e.Goal] = true } for m := range modules { for _, skill := range domain.CourseSkills { k := m + ":" + skill if coverage[k] != 20 || levels[k] != [3]int{8, 8, 4} || len(goals[k]) < 5 { return fmt.Errorf("invalid bank coverage %s", k) } } } return nil } func bankMember(value string, list []string) bool { for _, s := range list { if s == value { return true } } return false } func (s *AdaptiveStore) InstallBank(ctx context.Context) error { entries, err := BundledBank() if err != nil { return err } if _, err = s.db.ExecContext(ctx, bankSchema); err != nil { return err } tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() // Retire only templates. Assigned tasks and their frozen keys are never edited. if _, err = tx.ExecContext(ctx, `UPDATE course_v2_bank SET active=false WHERE language='en-US' AND version=$1`, domain.CourseVersion); err != nil { return err } for _, e := range entries { data, _ := json.Marshal(e) sum := sha256.Sum256(data) checksum := hex.EncodeToString(sum[:]) var old string err = tx.QueryRowContext(ctx, `SELECT checksum FROM course_v2_bank WHERE id=$1`, e.ID).Scan(&old) if err == nil && old != checksum { return fmt.Errorf("bank revision is immutable: %s", e.ID) } if err != nil && !errors.Is(err, sql.ErrNoRows) { return err } _, err = tx.ExecContext(ctx, `INSERT INTO course_v2_bank(id,language,version,revision,module_id,skill,goal,difficulty,data,checksum) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT(id) DO UPDATE SET active=true`, e.ID, e.Task.TargetLanguage, e.Task.CourseVersion, e.Revision, e.Task.ModuleID, e.Task.Skill, e.Goal, e.Difficulty, data, checksum) if err != nil { return err } } if err = tx.Commit(); err != nil { return err } return s.installVariants(ctx, entries) } type bankCandidate struct { entry domain.BankEntry bankEntryID string variantID string previousID string taskID string score float64 seen bool presented int64 } // AssignBankTask serializes selections per learner, including presentation of // unanswered tasks. Merely clicking Next is not a graded learning result. func (s *AdaptiveStore) AssignBankTask(ctx context.Context, j domain.CourseJob) (domain.AdaptiveTask, error) { tx, err := s.db.BeginTx(ctx, nil) if err != nil { return domain.AdaptiveTask{}, err } defer tx.Rollback() if _, err = tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1,0))`, j.UserID+":"+j.Language+":"+domain.CourseVersion); err != nil { return domain.AdaptiveTask{}, err } var result []byte err = tx.QueryRowContext(ctx, `SELECT result FROM course_v2_jobs WHERE id=$1 AND user_id=$2 AND lease=$3 AND status='running' FOR UPDATE`, j.ID, j.UserID, j.Lease).Scan(&result) if err != nil { return domain.AdaptiveTask{}, err } if len(result) > 0 { var old struct { TaskID string `json:"task_id"` Review bool `json:"review"` } if err = json.Unmarshal(result, &old); err != nil { return domain.AdaptiveTask{}, err } if old.TaskID != "" { if err = tx.Commit(); err != nil { return domain.AdaptiveTask{}, err } t, err := s.Task(ctx, j.UserID, j.Language, old.TaskID) t.Review = old.Review return t, err } } instructionLanguage := valueOr(j.InstructionLanguage, "de-DE") rows, err := tx.QueryContext(ctx, availableBank+` SELECT av.entry,COALESCE(t.id,''),COALESCE(graded.score,-1),COALESCE(extract(epoch from previous.last_presented_at)::bigint,0),av.base_id,av.variant_id,COALESCE(previous.id,'') FROM available av LEFT JOIN course_v2_tasks t ON t.bank_entry_id=av.base_id AND COALESCE(t.variant_id,'')=av.variant_id AND t.owner_id=$1 AND t.exam_id IS NULL LEFT JOIN LATERAL (SELECT r.id,r.last_presented_at FROM course_v2_tasks r WHERE r.owner_id=$1 AND r.language=$2 AND r.version=$3 AND r.exam_id IS NULL AND COALESCE(NULLIF(r.data->>'canonical_task_id',''),r.bank_entry_id)=av.canonical_id ORDER BY r.last_presented_at DESC NULLS LAST,r.id LIMIT 1) previous ON true LEFT JOIN LATERAL (SELECT (a.evaluation->>'score')::float8 AS score FROM course_v2_attempts a JOIN course_v2_tasks r ON r.id=a.task_id WHERE a.user_id=$1 AND r.owner_id=$1 AND r.language=$2 AND r.version=$3 AND r.exam_id IS NULL AND COALESCE(NULLIF(r.data->>'canonical_task_id',''),r.bank_entry_id)=av.canonical_id AND a.status='graded' AND NOT a.demo ORDER BY a.created_at DESC,a.id DESC LIMIT 1) graded ON true WHERE av.module_id=$5 AND ($6='' OR av.skill=$6) AND ($7=false OR av.entry->'task'->>'kind'='choice')`, j.UserID, j.Language, domain.CourseVersion, instructionLanguage, j.ModuleID, j.Skill, j.Kind == "legacy_next") if err != nil { return domain.AdaptiveTask{}, err } candidates := []bankCandidate{} for rows.Next() { var c bankCandidate var data []byte if err = rows.Scan(&data, &c.taskID, &c.score, &c.presented, &c.bankEntryID, &c.variantID, &c.previousID); err != nil { rows.Close() return domain.AdaptiveTask{}, err } if err = json.Unmarshal(data, &c.entry); err != nil { rows.Close() return domain.AdaptiveTask{}, err } c.seen = c.previousID != "" candidates = append(candidates, c) } err = rows.Err() rows.Close() if err != nil { return domain.AdaptiveTask{}, err } if len(candidates) == 0 { if instructionLanguage != "de-DE" { return domain.AdaptiveTask{}, ErrCourseVariantUnavailable } return domain.AdaptiveTask{}, ErrNotFound } type recent struct { goal string score float64 task string difficulty int } recentScores := []recent{} rows, err = tx.QueryContext(ctx, `SELECT b.goal,(a.evaluation->>'score')::float8,COALESCE(NULLIF(t.data->>'evidence_task_id',''),t.id),b.difficulty FROM course_v2_attempts a JOIN course_v2_tasks t ON t.id=a.task_id JOIN course_v2_bank b ON b.id=t.bank_entry_id WHERE a.user_id=$1 AND t.language=$2 AND t.version=$3 AND t.module_id=$4 AND ($5='' OR t.skill=$5) AND a.status='graded' AND NOT a.demo AND a.exam_id IS NULL ORDER BY a.created_at DESC,a.id DESC LIMIT 5`, j.UserID, j.Language, domain.CourseVersion, j.ModuleID, j.Skill) if err != nil { return domain.AdaptiveTask{}, err } for rows.Next() { var r recent if err = rows.Scan(&r.goal, &r.score, &r.task, &r.difficulty); err != nil { rows.Close() return domain.AdaptiveTask{}, err } recentScores = append(recentScores, r) } err = rows.Err() rows.Close() if err != nil { return domain.AdaptiveTask{}, err } target := 1 weak := map[string]bool{} distinct := map[string]bool{} sum := float64(0) for _, r := range recentScores { sum += r.score distinct[r.task] = true if r.score < 80 { weak[r.goal] = true } } if len(recentScores) > 0 { avg := sum / float64(len(recentScores)) target = recentScores[0].difficulty if len(distinct) >= 3 && avg >= 85 && target < 3 { target++ } else if len(distinct) >= 2 && avg < 60 && target > 1 { target-- } } var lastID string _ = tx.QueryRowContext(ctx, `SELECT id FROM course_v2_tasks WHERE owner_id=$1 AND language=$2 AND version=$3 AND module_id=$4 AND ($5='' OR skill=$5) AND exam_id IS NULL AND bank_entry_id IS NOT NULL ORDER BY last_presented_at DESC,id LIMIT 1`, j.UserID, j.Language, domain.CourseVersion, j.ModuleID, j.Skill).Scan(&lastID) rank := func(c bankCandidate) int { if !c.seen { if weak[c.entry.Goal] && c.entry.Difficulty <= target { return 0 } return 1 } if c.score >= 0 && c.score < 80 { return 2 } return 3 } sort.Slice(candidates, func(i, k int) bool { a, b := candidates[i], candidates[k] if a.seen && b.seen && (a.previousID == lastID) != (b.previousID == lastID) { return b.previousID == lastID } if rank(a) != rank(b) { return rank(a) < rank(b) } if a.seen && b.seen { if (a.previousID == lastID) != (b.previousID == lastID) { return b.previousID == lastID } if a.presented != b.presented { return a.presented < b.presented } } da, db := a.entry.Difficulty-target, b.entry.Difficulty-target if da < 0 { da = -da } if db < 0 { db = -db } if da != db { return da < db } ha := sha256.Sum256([]byte(j.UserID + a.entry.ID)) hb := sha256.Sum256([]byte(j.UserID + b.entry.ID)) return strings.Compare(hex.EncodeToString(ha[:]), hex.EncodeToString(hb[:])) < 0 }) chosen := candidates[0] t := chosen.entry.Task t.Source = "curated" t.Review = chosen.seen t.ID = chosen.taskID if chosen.taskID == "" { t.ID = NewID("a2") if t.InstructionLanguage == "" { t.InstructionLanguage = "de-DE" } if t.CanonicalTaskID == "" { t.CanonicalTaskID = chosen.bankEntryID } // Reuse the earliest evidence identity without editing any historic snapshot. err = tx.QueryRowContext(ctx, `SELECT COALESCE(NULLIF(data->>'evidence_task_id',''),id) FROM course_v2_tasks WHERE owner_id=$1 AND language=$2 AND version=$3 AND exam_id IS NULL AND COALESCE(NULLIF(data->>'canonical_task_id',''),bank_entry_id)=$4 ORDER BY created_at,id LIMIT 1`, j.UserID, j.Language, domain.CourseVersion, t.CanonicalTaskID).Scan(&t.EvidenceTaskID) if errors.Is(err, sql.ErrNoRows) { t.EvidenceTaskID = t.ID } else if err != nil { return t, err } data, _ := json.Marshal(t) _, err = tx.ExecContext(ctx, `INSERT INTO course_v2_tasks(id,module_id,language,version,owner_id,source,skill,kind,data,fingerprint,bank_entry_id,variant_id,last_presented_at,presentation_count) VALUES($1,$2,$3,$4,$5,'curated',$6,$7,$8,$9,$10,NULLIF($11,''),now(),1)`, t.ID, t.ModuleID, t.TargetLanguage, t.CourseVersion, j.UserID, t.Skill, t.Kind, data, "bank:"+chosen.entry.ID, chosen.bankEntryID, chosen.variantID) } else { _, err = tx.ExecContext(ctx, `UPDATE course_v2_tasks SET last_presented_at=now(),presentation_count=presentation_count+1 WHERE id=$1`, t.ID) } if err != nil { return domain.AdaptiveTask{}, err } data, _ := json.Marshal(map[string]any{"task_id": t.ID, "source": "curated", "review": chosen.seen}) if _, err = tx.ExecContext(ctx, `UPDATE course_v2_jobs SET result=$2 WHERE id=$1`, j.ID, data); err != nil { return domain.AdaptiveTask{}, err } if err = tx.Commit(); err != nil { return domain.AdaptiveTask{}, err } out, err := s.Task(ctx, j.UserID, j.Language, t.ID) out.Review = chosen.seen return out, err } func (s *AdaptiveStore) BankExam(ctx context.Context, user, language, module string) ([]domain.AdaptiveTask, error) { out := []domain.AdaptiveTask{} mods, err := s.Modules(ctx, language) if err != nil { return nil, err } if len(mods) != 7 { return nil, ErrNotFound } for i, skill := range domain.CourseSkills { for n := 0; n < 2; n++ { origin := mods[(i+n+1)%6].Key var data []byte err = s.db.QueryRowContext(ctx, `SELECT b.data FROM course_v2_bank b WHERE b.active AND b.language=$1 AND b.version=$2 AND b.module_id=$3 AND b.skill=$4 AND b.data->'task'->>'origin_module_key'=$5 ORDER BY (SELECT count(*) FROM course_v2_tasks t WHERE t.owner_id=$6 AND (t.bank_entry_id=b.id OR t.fingerprint='bank-exam:'||b.id)),md5(b.id||$6) LIMIT 1`, language, domain.CourseVersion, module, skill, origin, user).Scan(&data) if errors.Is(err, sql.ErrNoRows) { return nil, ErrNotFound } if err != nil { return nil, err } var entry domain.BankEntry if err = json.Unmarshal(data, &entry); err != nil { return nil, err } t := entry.Task t.Source = "curated" t.BriefID = entry.ID out = append(out, t) } } return out, nil } func (s *AdaptiveStore) Ungrade(ctx context.Context, user, id string) error { _, err := s.db.ExecContext(ctx, `UPDATE course_v2_attempts SET status='ungraded' WHERE id=$1 AND user_id=$2 AND status='pending'`, id, user) return err } type BankInventoryItem struct { ModuleID string `json:"module_id"` Skill string `json:"skill"` Total int `json:"total"` Unseen int `json:"unseen"` } func (s *AdaptiveStore) BankInventory(ctx context.Context, user, language string) ([]BankInventoryItem, error) { var instructionLanguage string if err := s.db.QueryRowContext(ctx, `SELECT instruction_language FROM users WHERE id=$1`, user).Scan(&instructionLanguage); err != nil { return nil, err } rows, err := s.db.QueryContext(ctx, availableBank+` SELECT av.module_id,av.skill,count(*),count(*) FILTER(WHERE NOT EXISTS(SELECT 1 FROM course_v2_tasks t WHERE t.owner_id=$1 AND t.language=$2 AND t.version=$3 AND t.exam_id IS NULL AND COALESCE(NULLIF(t.data->>'canonical_task_id',''),t.bank_entry_id)=av.canonical_id)) FROM available av GROUP BY av.module_id,av.skill ORDER BY av.module_id,av.skill`, user, language, domain.CourseVersion, instructionLanguage) if err != nil { return nil, err } defer rows.Close() out := []BankInventoryItem{} for rows.Next() { var item BankInventoryItem if err = rows.Scan(&item.ModuleID, &item.Skill, &item.Total, &item.Unseen); err != nil { return nil, err } out = append(out, item) } return out, rows.Err() } // Also repairs a crash between completing scores and storing the static summary. // Existing ready/historical summaries are not overwritten. func (s *AdaptiveStore) EnsureBankExamSummary(ctx context.Context, user, id string) error { _, err := s.db.ExecContext(ctx, `UPDATE course_v2_exams SET result=result||jsonb_build_object('summary',$3::text,'summary_status','ready') WHERE id=$1 AND user_id=$2 AND status='completed' AND result->>'summary_status'='pending'`, id, user, "Interne A1-Lerneinschätzung aus den gespeicherten Bewertungen. Kein Zertifikat.") return err }