Files
Eugenius Naumenko e274eaab2b Adaptiven A1-Kurs und statistisches Aussprachetraining integrieren
Direkte DeepSeek- und Mistral-Anbindung mit versionierten Aktionsprofilen,
Budgetreservierung, Tokenabrechnung und administrativen Qualitätsprüfungen
ersetzen die Flowise-Abhängigkeit im regulären Betrieb.

840 geprüfte A1-Aufgaben mit geschützter Testauswertung und konkreten offenen
Arbeitsaufträgen ergänzen den Kurs. Inhalts- und Sprachbewertung bleiben von
Azure-Aussprachewerten getrennt; historische Aufgaben und Antworten bleiben erhalten.

Echte Wort- und Phonemmessungen werden mit Herkunft, Aktualität und Nachweisdichte
gebündelt. Go bestimmt Trainingsziele und Wiederholungen, das Modell formuliert
dazu passende Nachsprechübungen. Profilansicht und Auswahlgründe werden angezeigt.

Validierung: Go-Race-Tests mit lokaler PostgreSQL-Integration, go vet,
Web-Build, Audit aller 840 Aufgaben und drei Browser-Smokes mit lokalen
Modell-/Speech-Fixtures bestanden. Keine Produktionsänderungen oder Live-Aufrufe.
2026-11-01 23:23:20 +00:00

673 lines
25 KiB
Go

package services
import (
"context"
"encoding/json"
"errors"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"speech-learning-api/internal/config"
"speech-learning-api/internal/domain"
"speech-learning-api/internal/store"
)
func aiTestProfile(action string) domain.AIProfile {
return domain.AIProfile{Action: action, Version: 1, Enabled: true, Provider: "mistral", Model: "fixture-model", MaxTokens: 2048, ContextTokens: 32768, TimeoutSeconds: 5, PricingSet: true, InputPrice: 1000000, CachedInputPrice: 100000, OutputPrice: 2000000}
}
func aiCompletion(content any, usage bool) map[string]any {
b, _ := json.Marshal(content)
r := map[string]any{"choices": []any{map[string]any{"finish_reason": "stop", "message": map[string]any{"content": string(b)}}}}
if usage {
r["usage"] = map[string]any{"prompt_tokens": 100, "completion_tokens": 20, "prompt_cache_hit_tokens": 40, "completion_tokens_details": map[string]any{"reasoning_tokens": 5}}
}
return r
}
func TestAICompletionAccountingAndContracts(t *testing.T) {
good := domain.CourseEvaluation{Score: 100, Verdict: "correct", Feedback: "Richtig", Issues: []string{}}
c := domain.AICall{Action: "evaluate_course_attempt", Profile: aiTestProfile("evaluate_course_attempt")}
b, _ := json.Marshal(aiCompletion(good, true))
var out domain.CourseEvaluation
if err := readAICompletion(b, &c, &out); err != nil {
t.Fatal(err)
}
// Reasoning is a subset of completion tokens; cache hits are a subset of input.
if c.Cost == nil || *c.Cost != 104 || *c.Usage.ReasoningTokens != 5 || c.Usage.OutputTokens != 20 {
t.Fatalf("double counting: %#v", c)
}
c = domain.AICall{Action: "evaluate_course_attempt", Profile: aiTestProfile("evaluate_course_attempt")}
b, _ = json.Marshal(aiCompletion(good, false))
if err := readAICompletion(b, &c, &out); err != nil || c.Cost != nil || c.Usage != nil {
t.Fatal("missing usage fabricated", err)
}
for _, content := range []any{map[string]any{"score": 100}, map[string]any{"verdict": "correct", "feedback": "Ja", "issues": []string{}}, map[string]any{"score": 100, "verdict": "correct", "feedback": "Ja", "issues": []string{}, "owner_id": "evil"}} {
c = domain.AICall{Action: "evaluate_course_attempt", Profile: aiTestProfile("evaluate_course_attempt")}
b, _ = json.Marshal(aiCompletion(content, true))
if readAICompletion(b, &c, &out) == nil {
t.Fatal("incomplete contract accepted")
}
if c.Cost == nil {
t.Fatal("invalid JSON lost billable usage")
}
}
raw := aiCompletion(good, true)
raw["choices"] = []any{map[string]any{"finish_reason": "length", "message": map[string]any{"content": "{\"score\":"}}}
b, _ = json.Marshal(raw)
c = domain.AICall{Profile: aiTestProfile("evaluate_course_attempt")}
if readAICompletion(b, &c, &out) == nil || c.Cost == nil {
t.Fatal("truncated reply or cost lost")
}
}
func TestAIProfilesAndPromptIsolation(t *testing.T) {
a := NewAIService(nil, config.Config{})
if len(a.Prompts) != len(domain.AIActions) || a.PromptVersion == "" {
t.Fatal("missing embedded prompts")
}
eval := a.Prompts["evaluate_course_attempt"]
if strings.Contains(eval, "Für task=\"plan_course_task\"") {
t.Fatal("mixed task prompt")
}
if !strings.Contains(eval, "Punktabzug") || !strings.Contains(a.Prompts["analyze_course_progress"], "distinct_tasks") {
t.Fatal("quality regressions missing")
}
p := aiTestProfile("evaluate_course_attempt")
if err := ValidAIProfile(p); err != nil {
t.Fatal(err)
}
p.PricingSet = false
if ValidAIProfile(p) == nil {
t.Fatal("unpriced enabled model")
}
p.PricingSet = true
p.Provider = "deepseek"
p.JSONSchema = true
if ValidAIProfile(p) == nil {
t.Fatal("unsupported DeepSeek schema mode")
}
}
func TestMistralTaskSchemasKeepRendererRequirements(t *testing.T) {
a := NewAIService(nil, config.Config{})
for _, test := range []struct{ skill, kind, required, absent string }{
{"grammar", "choice", "options", "audio_text"},
{"listening", "choice", "audio_text", "passage"},
{"reading", "short_text", "passage", "options"},
{"speaking", "repeat_sentence", "reference_text", "options"},
{"speaking", "free_speech", "answer_key", "reference_text"},
{"writing", "free_text", "answer_key", "options"},
} {
t.Run(test.skill+"/"+test.kind, func(t *testing.T) {
var full map[string]any
if err := json.Unmarshal([]byte(a.Schemas["author_course_task"]), &full); err != nil {
t.Fatal(err)
}
before, _ := json.Marshal(full)
schema := mistralStructuralSchema(full)
constrainTaskSchema(schema, "author_course_task", map[string]any{"brief": map[string]any{"module_id": "a1_family_v1", "skill": test.skill, "kind": test.kind}})
after, _ := json.Marshal(full)
if string(before) != string(after) {
t.Fatal("persisted contract was mutated")
}
props := schema.(map[string]any)["properties"].(map[string]any)
tasks := props["tasks"].(map[string]any)
item := tasks["items"].(map[string]any)
fields := item["properties"].(map[string]any)
if tasks["minItems"] != 1 || tasks["maxItems"] != 1 || item["allOf"] != nil || fields[test.absent] != nil {
t.Fatal("wrong decoder contract", tasks)
}
found := false
for _, field := range item["required"].([]any) {
found = found || field == test.required
}
if !found || fields["kind"].(map[string]any)["const"] != test.kind {
t.Fatal("required renderer fields lost")
}
})
}
}
func aiChoiceCheck(task domain.AdaptiveTask, index int, validIDs ...string) AIChoiceCheck {
check := AIChoiceCheck{TaskIndex: index}
for _, option := range task.Options {
valid := false
for _, id := range validIDs {
valid = valid || id == option.ID
}
check.Options = append(check.Options, AIOptionCheck{OptionID: option.ID, IsCorrect: valid, Reason: "Test reasoning"})
}
return check
}
func TestIndependentReviewRejectsAnchoredAndAmbiguousSolutions(t *testing.T) {
tasks := []domain.AdaptiveTask{store.EnglishCoursePack().Reserves[7]}
key := tasks[0].AnswerKey.CorrectOptionID
blind := blindReviewTasks(tasks)
if blind[0].AnswerKey != nil || tasks[0].AnswerKey == nil {
t.Fatal("solution leaked or original mutated")
}
good := AIReview{SchemaVersion: 2, Approved: true, ChoiceChecks: []AIChoiceCheck{aiChoiceCheck(tasks[0], 0, key)}}
if err := validateIndependentReview(tasks, good); err != nil {
t.Fatal(err)
}
wrong := aiChoiceCheck(tasks[0], 0, tasks[0].Options[1].ID)
ambiguous := aiChoiceCheck(tasks[0], 0, key, tasks[0].Options[1].ID)
missing := aiChoiceCheck(tasks[0], 0, key)
missing.Options = missing.Options[:1]
duplicate := aiChoiceCheck(tasks[0], 0, key)
duplicate.Options[1].OptionID = key
outside := aiChoiceCheck(tasks[0], 1, key)
for _, checks := range [][]AIChoiceCheck{nil, {wrong}, {ambiguous}, {missing}, {duplicate}, {outside}, {good.ChoiceChecks[0], good.ChoiceChecks[0]}} {
bad := good
bad.ChoiceChecks = checks
if validateIndependentReview(tasks, bad) == nil {
t.Fatal("unproven solution accepted", checks)
}
}
}
func setupAI(t *testing.T) (*AIService, *store.PostgresStore, string) {
t.Helper()
db, profiles := adaptiveTestDB(t)
repo := store.NewAIStore(db)
user, err := profiles.CreateUser(context.Background(), store.NewID("ai")+"@example.test", "hash", "de-DE")
if err != nil {
t.Fatal(err)
}
a := NewAIService(repo, config.Config{MistralAPIKey: "fixture-secret", DeepSeekAPIKey: "fixture-deepseek"})
return a, profiles, user.ID
}
func TestAIPostgresSnapshotsUsageAndQuality(t *testing.T) {
a, _, user := setupAI(t)
ctx := context.Background()
p := aiTestProfile("evaluate_course_attempt")
if _, err := a.Repo.SaveProfile(ctx, user, p); err != nil {
t.Fatal(err)
}
if err := a.Repo.SaveBudget(ctx, user, 1000000, 0); err != nil {
t.Fatal(err)
}
var count atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
count.Add(1)
if r.Header.Get("Authorization") != "Bearer fixture-secret" {
t.Error("missing credential")
}
var payload struct {
Model string `json:"model"`
Messages []struct {
Role string `json:"role"`
Content string `json:"content"`
} `json:"messages"`
}
json.NewDecoder(r.Body).Decode(&payload)
if payload.Model != "fixture-model" {
t.Error("changed in-flight model")
}
if len(payload.Messages) != 2 || !strings.Contains(payload.Messages[0].Content, "Punktabzug") {
t.Error("wrong system prompt")
}
var input struct {
Task string `json:"task"`
Input struct {
Answer string `json:"answer"`
} `json:"input"`
}
json.Unmarshal([]byte(payload.Messages[1].Content), &input)
good := !strings.Contains(input.Input.Answer, "He have")
evaluation := domain.CourseEvaluation{Score: 100, Verdict: "correct", Feedback: "Richtig", Issues: []string{}}
if !good {
evaluation = domain.CourseEvaluation{Score: 60, Verdict: "partial", Feedback: "He braucht has got.", Issues: []string{"Falsche Verbform have nach he"}}
}
json.NewEncoder(w).Encode(aiCompletion(evaluation, true))
}))
defer server.Close()
a.URLs["mistral"] = server.URL
ctx, o, err := a.Begin(ctx, user, "quality-operation", true)
if err != nil {
t.Fatal(err)
}
// Saving another profile while the operation runs must not change its model or price.
p.Version = 2
p.Model = "new-model"
p.OutputPrice = 9000000
if _, err = a.Repo.SaveProfile(ctx, user, p); err != nil {
t.Fatal(err)
}
results, err := a.Quality(ctx)
if err != nil || len(results) != 2 {
t.Fatal(err)
}
for _, r := range results {
if !r.Passed {
t.Fatalf("quality failed: %#v", r)
}
}
if _, err = a.Quality(ctx); err != nil {
t.Fatal("cached quality", err)
}
if count.Load() != 2 {
t.Fatal("repeated request billed twice")
}
report, err := a.Repo.Usage(ctx, domain.AIUsageFilter{User: user})
if err != nil || report.Total != 2 || report.InputTokens != 200 || report.OutputTokens != 40 || report.EstimatedCost != 208 {
t.Fatalf("report %#v: %v", report, err)
}
for _, c := range report.Calls {
if c.Profile.Version != 2 || c.Profile.Model != "fixture-model" || !c.Test || c.Input != "" {
t.Fatalf("wrong frozen metadata: %#v", c)
}
detail, err := a.Repo.Call(ctx, c.ID)
if err != nil || detail.Output == "" || strings.Contains(detail.Input, "fixture-secret") {
t.Fatal("unsafe/missing detail", err)
}
}
restarted, _, err := a.Begin(context.Background(), user, o.ID, true)
if err != nil {
t.Fatal(err)
}
if _, err = a.Quality(restarted); err != nil || count.Load() != 2 {
t.Fatal("restart ignored saved results", err)
}
}
func TestAIPostgresConcurrentBudgetAndUnknown(t *testing.T) {
a, _, user := setupAI(t)
ctx := context.Background()
repo := a.Repo
if err := repo.SaveBudget(ctx, user, 100, 0); err != nil {
t.Fatal(err)
}
success := atomic.Int32{}
var wg sync.WaitGroup
for i := 0; i < 8; i++ {
wg.Add(1)
go func() {
defer wg.Done()
id := store.NewID("budget")
_, err := repo.Operation(ctx, domain.AIOperation{ID: id, UserID: user})
if err != nil {
t.Error(err)
return
}
_, err = repo.Reserve(ctx, domain.AICall{OperationID: id, UserID: user, Action: "evaluate_course_attempt", Reserved: 60, Profile: aiTestProfile("evaluate_course_attempt")})
if err == nil {
success.Add(1)
} else if !errors.Is(err, store.ErrAIBudget) {
t.Error(err)
}
}()
}
wg.Wait()
if success.Load() != 1 {
t.Fatalf("budget raced: %d", success.Load())
}
budget, err := repo.Budget(ctx)
if err != nil || budget.MonthCommitted != 60 || budget.OpenReservations != 60 {
t.Fatal(budget, err)
}
report, _ := repo.Usage(ctx, domain.AIUsageFilter{})
c := report.Calls[0]
c.Status = "unknown"
if err = repo.Finish(ctx, c); err != nil {
t.Fatal(err)
}
if err = repo.Resolve(ctx, user, c.ID, 10, "Provider invoice reconciled"); err != nil {
t.Fatal(err)
}
budget, _ = repo.Budget(ctx)
if budget.MonthCommitted != 10 || budget.OpenReservations != 0 {
t.Fatal("reservation released incorrectly", budget)
}
c, _ = repo.Call(ctx, c.ID)
if !strings.Contains(c.Error, user) {
t.Fatal("reconciliation unaudited")
}
}
func TestAIPostgresProviderFailuresAndUnknownUsage(t *testing.T) {
for _, scenario := range []string{"missing_usage", "length", "unauthorized", "rate_limit", "disconnect"} {
t.Run(scenario, func(t *testing.T) {
a, _, user := setupAI(t)
ctx := context.Background()
p := aiTestProfile("evaluate_course_attempt")
p.Provider = "deepseek"
p.ReasoningEffort = "none"
if _, err := a.Repo.SaveProfile(ctx, user, p); err != nil {
t.Fatal(err)
}
a.Repo.SaveBudget(ctx, user, 1000000, 0)
var calls atomic.Int32
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls.Add(1)
var payload map[string]any
json.NewDecoder(r.Body).Decode(&payload)
if _, ok := payload["reasoning_effort"]; ok {
t.Error("none effort leaked to DeepSeek")
}
if payload["thinking"].(map[string]any)["type"] != "disabled" {
t.Error("thinking not disabled")
}
if scenario == "unauthorized" {
w.WriteHeader(401)
io.WriteString(w, `{"error":{"message":"Invalid fixture-secret"}}`)
return
}
if scenario == "rate_limit" {
w.WriteHeader(429)
io.WriteString(w, `{"message":"Rate limit exceeded fixture-secret"}`)
return
}
if scenario == "disconnect" {
conn, _, _ := w.(http.Hijacker).Hijack()
conn.Close()
return
}
result := aiCompletion(domain.CourseEvaluation{Score: 100, Verdict: "correct", Feedback: "Richtig", Issues: []string{}}, scenario != "missing_usage")
if scenario == "length" {
result["choices"] = []any{map[string]any{"finish_reason": "length", "message": map[string]any{"content": "{"}}}
}
json.NewEncoder(w).Encode(result)
}))
defer server.Close()
a.URLs["deepseek"] = server.URL
ctx, _, err := a.Begin(ctx, user, store.NewID("failure"), true)
if err != nil {
t.Fatal(err)
}
var evaluation domain.CourseEvaluation
err = a.Call(ctx, "evaluate_course_attempt", map[string]any{"answer": "He has got a sister."}, &evaluation)
report, _ := a.Repo.Usage(ctx, domain.AIUsageFilter{})
switch scenario {
case "missing_usage":
if err != nil || report.Unknown != 1 || report.Reserved == 0 {
t.Fatal("missing usage treated as zero", err, report)
}
case "length":
if err == nil || calls.Load() != 2 || report.Total != 2 || report.EstimatedCost == 0 {
t.Fatal("invalid completion not charged or unbounded retries", err, report)
}
case "unauthorized":
if err == nil || calls.Load() != 1 || report.Reserved != 0 || report.EstimatedCost != 0 {
t.Fatal("rejected request billing", err, report)
}
if strings.Contains(err.Error(), "fixture-secret") || !strings.Contains(err.Error(), "Invalid [redacted]") {
t.Fatal("provider message missing or credential exposed", err)
}
case "rate_limit":
if err == nil || calls.Load() != 1 || report.Reserved != 0 || report.EstimatedCost != 0 || !strings.Contains(err.Error(), "HTTP 429 : Rate limit exceeded [redacted]") || strings.Contains(err.Error(), "fixture-secret") {
t.Fatal("rate limit reason or zero billing lost", err, report)
}
case "disconnect":
if err == nil || calls.Load() != 1 || report.Reserved == 0 {
t.Fatal("uncertain bill lost", err, report)
}
}
})
}
}
func TestAIPostgresDirectCoursePipeline(t *testing.T) {
db, profiles := adaptiveTestDB(t)
account, err := profiles.CreateUser(context.Background(), "direct-course@example.test", "hash", "de-DE")
if err != nil {
t.Fatal(err)
}
user := account.ID
a := NewAIService(store.NewAIStore(db), config.Config{MistralAPIKey: "fixture-secret"})
ctx := context.Background()
profiles.CreateLearnerProfile(ctx, user, "en-US", "A1", 10)
for _, action := range []string{"analyze_course_progress", "plan_course_task", "author_course_task", "review_course_task", "next_exercise"} {
if _, err := a.Repo.SaveProfile(ctx, user, aiTestProfile(action)); err != nil {
t.Fatal(err)
}
}
a.Repo.SaveBudget(ctx, user, 10000000, 0)
var authors atomic.Int32
var candidate any
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var body struct {
Messages []struct {
Content string `json:"content"`
} `json:"messages"`
}
json.NewDecoder(r.Body).Decode(&body)
var q struct {
Task string `json:"task"`
Input json.RawMessage `json:"input"`
}
json.Unmarshal([]byte(body.Messages[1].Content), &q)
var result any
switch q.Task {
case "analyze_course_progress":
mods := []domain.ModuleAnalysis{}
for _, m := range store.EnglishCoursePack().Modules {
mods = append(mods, domain.ModuleAnalysis{ModuleID: m.ID, Strengths: []string{}, Weaknesses: []string{}, Recommendation: "Bereich ausprobieren", Reason: "Noch keine Nachweise", EvidenceIDs: []string{}})
}
result = domain.CourseAnalysis{SchemaVersion: 2, TargetLanguage: "en-US", CourseVersion: domain.CourseVersion, Summary: "Noch keine Nachweise.", RecommendedModuleID: "a1_introductions_v1", Modules: mods}
case "plan_course_task":
result = domain.CourseBrief{SchemaVersion: 2, ModuleID: "a1_family_v1", TargetLanguage: "en-US", CourseVersion: domain.CourseVersion, Skill: "grammar", Kind: "choice", Objective: "have und has unterscheiden", Difficulty: "A1", Reason: "Einstieg"}
case "author_course_task":
authors.Add(1)
task := store.EnglishCoursePack().Reserves[7]
task.ID = ""
task.Source = ""
candidate = map[string]any{"schema_version": 2, "tasks": []domain.AdaptiveTask{task}}
result = candidate
case "review_course_task":
var input struct {
Candidate struct {
Tasks []domain.AdaptiveTask `json:"tasks"`
} `json:"candidate"`
}
json.Unmarshal(q.Input, &input)
if len(input.Candidate.Tasks) != 1 {
t.Error("reviewer missing author output")
}
if input.Candidate.Tasks[0].AnswerKey != nil {
t.Error("reviewer was shown the authored solution")
}
result = AIReview{SchemaVersion: 2, Approved: true, ChoiceChecks: []AIChoiceCheck{aiChoiceCheck(store.EnglishCoursePack().Reserves[7], 0, store.EnglishCoursePack().Reserves[7].AnswerKey.CorrectOptionID)}, ReviewNotes: "Unabhängig gelöst"}
case "next_exercise":
result = map[string]any{"exercise_type": "repeat_sentence", "target_language": "en-US", "cefr_level": "A1", "reference_text": "My sister is here.", "display_prompt": "Sprich nach.", "tts_text": "My sister is here.", "focus_points": []string{"sister"}, "success_criteria": []string{"verständlich sprechen"}, "reason_for_choice": "Familienwort"}
default:
t.Errorf("unknown operation %s", q.Task)
}
json.NewEncoder(w).Encode(aiCompletion(result, true))
}))
defer server.Close()
a.URLs["mistral"] = server.URL
engine := NewCourseEngine(store.NewAdaptiveStore(db), profiles, a, slog.New(slog.NewTextHandler(io.Discard, nil)))
job, err := engine.Repo.Enqueue(ctx, user, "en-US", "next", "a1_family_v1", "grammar", "", "direct-request")
if err != nil {
t.Fatal(err)
}
claimed, err := engine.Repo.Claim(ctx)
if err != nil || claimed.ID != job.ID {
t.Fatal(err)
}
generated, err := engine.Process(ctx, claimed)
if err != nil || generated.Source != "ai" || authors.Load() != 1 {
t.Fatal(generated, err)
}
stored, err := engine.Repo.Task(ctx, user, "en-US", generated.ID)
if err != nil || stored.Source != "ai" || stored.AnswerKey == nil {
t.Fatal("course provenance/storage", stored, err)
}
if err = engine.Repo.Finish(ctx, claimed, generated, ""); err != nil {
t.Fatal(err)
}
exercise, err := a.NextExercise(ctx, domain.GeneratedExerciseRequest{UserID: user, TargetLanguage: "en-US", CEFRLevel: "A1"})
if err != nil || exercise.Source != "ai" {
t.Fatal(exercise, err)
}
if _, saved, err := profiles.SaveGeneratedExercise(ctx, user, exercise); err != nil || saved.Source != "ai" {
t.Fatal("pronunciation persistence", saved, err)
}
}
// A response timeout is intentionally not retried blindly: billing may have
// occurred, and the remaining budget reservation must remain visible.
func TestAICallDeadlineWithoutNetwork(t *testing.T) {
a, _, user := setupAI(t)
ctx := context.Background()
a.Repo.SaveProfile(ctx, user, aiTestProfile("evaluate_course_attempt"))
a.Repo.SaveBudget(ctx, user, 1000000, 0)
a.Client = &http.Client{Transport: speechRoundTripFunc(func(r *http.Request) (*http.Response, error) { return nil, context.DeadlineExceeded })}
work, _, err := a.Begin(ctx, user, "deadline", true)
if err != nil {
t.Fatal(err)
}
var out domain.CourseEvaluation
start := time.Now()
if err = a.Call(work, "evaluate_course_attempt", map[string]string{"answer": "Hello"}, &out); err == nil {
t.Fatal("timeout ignored")
}
if time.Since(start) > time.Second {
t.Fatal("unexpected wait")
}
report, _ := a.Repo.Usage(ctx, domain.AIUsageFilter{})
if report.Total != 1 || report.Unknown != 1 || report.Reserved == 0 {
t.Fatal(report)
}
}
func TestAIPostgresDailyLimitAndUnknownResolution(t *testing.T) {
a, profiles, user := setupAI(t)
ctx := context.Background()
repo := a.Repo
p := aiTestProfile("evaluate_course_attempt")
saved, err := repo.SaveProfile(ctx, user, p)
if err != nil {
t.Fatal(err)
}
if _, err = repo.SaveProfile(ctx, user, p); !errors.Is(err, store.ErrAIConflict) {
t.Fatal("stale settings overwrote profile", err)
}
if err = profiles.Init(ctx); err != nil {
t.Fatal(err)
}
all, err := repo.Profiles(ctx)
if err != nil || all[p.Action].Version != saved.Version || !all[p.Action].Enabled {
t.Fatal("migration reset settings", err)
}
if err = repo.SaveBudget(ctx, user, 1000000, 100); err != nil {
t.Fatal(err)
}
reserve := func(id string) (domain.AICall, error) {
_, err := repo.Operation(ctx, domain.AIOperation{ID: id, UserID: user})
if err != nil {
return domain.AICall{}, err
}
return repo.Reserve(ctx, domain.AICall{OperationID: id, UserID: user, Action: p.Action, Profile: saved, Reserved: 60})
}
c, err := reserve("daily-one")
if err != nil {
t.Fatal(err)
}
if _, err = reserve("daily-two"); !errors.Is(err, store.ErrAIBudget) {
t.Fatal("daily limit ignored", err)
}
c.Status = "succeeded"
c.Output = `{"score":100,"verdict":"correct","feedback":"Gut","issues":[]}`
if err = repo.Finish(ctx, c); err != nil {
t.Fatal(err)
}
// Successful output with missing usage remains cached and can be reconciled.
if err = repo.Resolve(ctx, user, c.ID, 10, "Abgleich mit Anbieterrechnung"); err != nil {
t.Fatal(err)
}
if err = repo.Resolve(ctx, user, c.ID, 0, "Erneuter Abgleich"); !errors.Is(err, store.ErrAIConflict) {
t.Fatal("reconciliation overwritten", err)
}
cached, err := repo.Cached(ctx, c.OperationID, c.Action, c.InputHash)
if err != nil || cached == nil || cached.Cost == nil || *cached.Cost != 10 {
t.Fatal("reconciliation lost cached output", err)
}
if _, err = reserve("daily-two"); err != nil {
t.Fatal("released budget unavailable", err)
}
}
func TestAIPostgresExplicitRetryRefreshesFrozenProfile(t *testing.T) {
db, profiles := adaptiveTestDB(t)
ctx := context.Background()
account, err := profiles.CreateUser(ctx, "retry-ai@example.test", "hash", "de-DE")
if err != nil {
t.Fatal(err)
}
a := NewAIService(store.NewAIStore(db), config.Config{MistralAPIKey: "fixture-secret"})
engine := NewCourseEngine(store.NewAdaptiveStore(db), profiles, a, slog.New(slog.NewTextHandler(io.Discard, nil)))
task, err := engine.Repo.Reserve(ctx, account.ID, "en-US", "a1_family_v1", "writing")
if err != nil {
t.Fatal(err)
}
attempt, job, err := engine.Submit(ctx, account.ID, "en-US", task.ID, "retry-saved-answer", "He has got a sister.", nil)
if err != nil || job == nil {
t.Fatal(err)
}
claimed, err := engine.Repo.Claim(ctx)
if err != nil {
t.Fatal(err)
}
if _, err = engine.Process(ctx, claimed); !errors.Is(err, store.ErrAIDisabled) {
t.Fatal("disabled profile not detected", err)
}
engine.Repo.FailAttempt(ctx, account.ID, attempt.ID)
engine.Repo.Finish(ctx, claimed, nil, "disabled")
p := aiTestProfile("evaluate_course_attempt")
if _, err = a.Repo.SaveProfile(ctx, account.ID, p); err != nil {
t.Fatal(err)
}
a.Repo.SaveBudget(ctx, account.ID, 1000000, 0)
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
json.NewEncoder(w).Encode(aiCompletion(domain.CourseEvaluation{Score: 100, Verdict: "correct", Feedback: "Richtig", Issues: []string{}}, true))
}))
defer provider.Close()
a.URLs["mistral"] = provider.URL
retried, err := engine.Repo.RetryEvaluation(ctx, account.ID, "en-US", attempt.ID)
if err != nil || retried.AIRevision != 1 {
t.Fatal("new retry revision missing", retried, err)
}
claimed, err = engine.Repo.Claim(ctx)
if err != nil || claimed.ID != job.ID {
t.Fatal(err)
}
if _, err = engine.Process(ctx, claimed); err != nil {
t.Fatal("retry retained unusable old settings", err)
}
preserved, err := engine.Repo.Attempt(ctx, account.ID, "en-US", attempt.ID)
if err != nil || preserved.Answer != "He has got a sister." || preserved.Evaluation == nil || preserved.Evaluation.Score != 100 {
t.Fatal("answer lost during retry", preserved, err)
}
}
func TestAIPostgresOperationDoesNotMutateSharedPrompts(t *testing.T) {
a, _, user := setupAI(t)
ctx := context.Background()
action := "evaluate_course_attempt"
old := a.Prompts[action]
if _, _, err := a.Begin(ctx, user, "frozen-prompts", true); err != nil {
t.Fatal(err)
}
a.Prompts[action] = "new prompt version"
a.PromptVersion = "new-version"
var wg sync.WaitGroup
for i := 0; i < 8; i++ {
wg.Add(1)
go func() {
defer wg.Done()
_, frozen, err := a.Begin(ctx, user, "frozen-prompts", true)
if err != nil || frozen.Prompts[action] != old {
t.Error("old prompt snapshot lost", err)
}
}()
}
wg.Wait()
if a.Prompts[action] != "new prompt version" {
t.Fatal("reading old operation overwrote shared application prompts")
}
}