Adaptiven A1-Kurs und statistisches Aussprachetraining integrieren
Direkte DeepSeek- und Mistral-Anbindung mit versionierten Aktionsprofilen, Budgetreservierung, Tokenabrechnung und administrativen Qualitätsprüfungen ersetzen die Flowise-Abhängigkeit im regulären Betrieb. 840 geprüfte A1-Aufgaben mit geschützter Testauswertung und konkreten offenen Arbeitsaufträgen ergänzen den Kurs. Inhalts- und Sprachbewertung bleiben von Azure-Aussprachewerten getrennt; historische Aufgaben und Antworten bleiben erhalten. Echte Wort- und Phonemmessungen werden mit Herkunft, Aktualität und Nachweisdichte gebündelt. Go bestimmt Trainingsziele und Wiederholungen, das Modell formuliert dazu passende Nachsprechübungen. Profilansicht und Auswahlgründe werden angezeigt. Validierung: Go-Race-Tests mit lokaler PostgreSQL-Integration, go vet, Web-Build, Audit aller 840 Aufgaben und drei Browser-Smokes mit lokalen Modell-/Speech-Fixtures bestanden. Keine Produktionsänderungen oder Live-Aufrufe.
This commit is contained in:
@@ -0,0 +1,672 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"speech-learning-api/internal/config"
|
||||
"speech-learning-api/internal/domain"
|
||||
"speech-learning-api/internal/store"
|
||||
)
|
||||
|
||||
func aiTestProfile(action string) domain.AIProfile {
|
||||
return domain.AIProfile{Action: action, Version: 1, Enabled: true, Provider: "mistral", Model: "fixture-model", MaxTokens: 2048, ContextTokens: 32768, TimeoutSeconds: 5, PricingSet: true, InputPrice: 1000000, CachedInputPrice: 100000, OutputPrice: 2000000}
|
||||
}
|
||||
func aiCompletion(content any, usage bool) map[string]any {
|
||||
b, _ := json.Marshal(content)
|
||||
r := map[string]any{"choices": []any{map[string]any{"finish_reason": "stop", "message": map[string]any{"content": string(b)}}}}
|
||||
if usage {
|
||||
r["usage"] = map[string]any{"prompt_tokens": 100, "completion_tokens": 20, "prompt_cache_hit_tokens": 40, "completion_tokens_details": map[string]any{"reasoning_tokens": 5}}
|
||||
}
|
||||
return r
|
||||
}
|
||||
func TestAICompletionAccountingAndContracts(t *testing.T) {
|
||||
good := domain.CourseEvaluation{Score: 100, Verdict: "correct", Feedback: "Richtig", Issues: []string{}}
|
||||
c := domain.AICall{Action: "evaluate_course_attempt", Profile: aiTestProfile("evaluate_course_attempt")}
|
||||
b, _ := json.Marshal(aiCompletion(good, true))
|
||||
var out domain.CourseEvaluation
|
||||
if err := readAICompletion(b, &c, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Reasoning is a subset of completion tokens; cache hits are a subset of input.
|
||||
if c.Cost == nil || *c.Cost != 104 || *c.Usage.ReasoningTokens != 5 || c.Usage.OutputTokens != 20 {
|
||||
t.Fatalf("double counting: %#v", c)
|
||||
}
|
||||
c = domain.AICall{Action: "evaluate_course_attempt", Profile: aiTestProfile("evaluate_course_attempt")}
|
||||
b, _ = json.Marshal(aiCompletion(good, false))
|
||||
if err := readAICompletion(b, &c, &out); err != nil || c.Cost != nil || c.Usage != nil {
|
||||
t.Fatal("missing usage fabricated", err)
|
||||
}
|
||||
for _, content := range []any{map[string]any{"score": 100}, map[string]any{"verdict": "correct", "feedback": "Ja", "issues": []string{}}, map[string]any{"score": 100, "verdict": "correct", "feedback": "Ja", "issues": []string{}, "owner_id": "evil"}} {
|
||||
c = domain.AICall{Action: "evaluate_course_attempt", Profile: aiTestProfile("evaluate_course_attempt")}
|
||||
b, _ = json.Marshal(aiCompletion(content, true))
|
||||
if readAICompletion(b, &c, &out) == nil {
|
||||
t.Fatal("incomplete contract accepted")
|
||||
}
|
||||
if c.Cost == nil {
|
||||
t.Fatal("invalid JSON lost billable usage")
|
||||
}
|
||||
}
|
||||
raw := aiCompletion(good, true)
|
||||
raw["choices"] = []any{map[string]any{"finish_reason": "length", "message": map[string]any{"content": "{\"score\":"}}}
|
||||
b, _ = json.Marshal(raw)
|
||||
c = domain.AICall{Profile: aiTestProfile("evaluate_course_attempt")}
|
||||
if readAICompletion(b, &c, &out) == nil || c.Cost == nil {
|
||||
t.Fatal("truncated reply or cost lost")
|
||||
}
|
||||
}
|
||||
func TestAIProfilesAndPromptIsolation(t *testing.T) {
|
||||
a := NewAIService(nil, config.Config{})
|
||||
if len(a.Prompts) != len(domain.AIActions) || a.PromptVersion == "" {
|
||||
t.Fatal("missing embedded prompts")
|
||||
}
|
||||
eval := a.Prompts["evaluate_course_attempt"]
|
||||
if strings.Contains(eval, "Für task=\"plan_course_task\"") {
|
||||
t.Fatal("mixed task prompt")
|
||||
}
|
||||
if !strings.Contains(eval, "Punktabzug") || !strings.Contains(a.Prompts["analyze_course_progress"], "distinct_tasks") {
|
||||
t.Fatal("quality regressions missing")
|
||||
}
|
||||
p := aiTestProfile("evaluate_course_attempt")
|
||||
if err := ValidAIProfile(p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p.PricingSet = false
|
||||
if ValidAIProfile(p) == nil {
|
||||
t.Fatal("unpriced enabled model")
|
||||
}
|
||||
p.PricingSet = true
|
||||
p.Provider = "deepseek"
|
||||
p.JSONSchema = true
|
||||
if ValidAIProfile(p) == nil {
|
||||
t.Fatal("unsupported DeepSeek schema mode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMistralTaskSchemasKeepRendererRequirements(t *testing.T) {
|
||||
a := NewAIService(nil, config.Config{})
|
||||
for _, test := range []struct{ skill, kind, required, absent string }{
|
||||
{"grammar", "choice", "options", "audio_text"},
|
||||
{"listening", "choice", "audio_text", "passage"},
|
||||
{"reading", "short_text", "passage", "options"},
|
||||
{"speaking", "repeat_sentence", "reference_text", "options"},
|
||||
{"speaking", "free_speech", "answer_key", "reference_text"},
|
||||
{"writing", "free_text", "answer_key", "options"},
|
||||
} {
|
||||
t.Run(test.skill+"/"+test.kind, func(t *testing.T) {
|
||||
var full map[string]any
|
||||
if err := json.Unmarshal([]byte(a.Schemas["author_course_task"]), &full); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, _ := json.Marshal(full)
|
||||
schema := mistralStructuralSchema(full)
|
||||
constrainTaskSchema(schema, "author_course_task", map[string]any{"brief": map[string]any{"module_id": "a1_family_v1", "skill": test.skill, "kind": test.kind}})
|
||||
after, _ := json.Marshal(full)
|
||||
if string(before) != string(after) {
|
||||
t.Fatal("persisted contract was mutated")
|
||||
}
|
||||
props := schema.(map[string]any)["properties"].(map[string]any)
|
||||
tasks := props["tasks"].(map[string]any)
|
||||
item := tasks["items"].(map[string]any)
|
||||
fields := item["properties"].(map[string]any)
|
||||
if tasks["minItems"] != 1 || tasks["maxItems"] != 1 || item["allOf"] != nil || fields[test.absent] != nil {
|
||||
t.Fatal("wrong decoder contract", tasks)
|
||||
}
|
||||
found := false
|
||||
for _, field := range item["required"].([]any) {
|
||||
found = found || field == test.required
|
||||
}
|
||||
if !found || fields["kind"].(map[string]any)["const"] != test.kind {
|
||||
t.Fatal("required renderer fields lost")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func aiChoiceCheck(task domain.AdaptiveTask, index int, validIDs ...string) AIChoiceCheck {
|
||||
check := AIChoiceCheck{TaskIndex: index}
|
||||
for _, option := range task.Options {
|
||||
valid := false
|
||||
for _, id := range validIDs {
|
||||
valid = valid || id == option.ID
|
||||
}
|
||||
check.Options = append(check.Options, AIOptionCheck{OptionID: option.ID, IsCorrect: valid, Reason: "Test reasoning"})
|
||||
}
|
||||
return check
|
||||
}
|
||||
func TestIndependentReviewRejectsAnchoredAndAmbiguousSolutions(t *testing.T) {
|
||||
tasks := []domain.AdaptiveTask{store.EnglishCoursePack().Reserves[7]}
|
||||
key := tasks[0].AnswerKey.CorrectOptionID
|
||||
blind := blindReviewTasks(tasks)
|
||||
if blind[0].AnswerKey != nil || tasks[0].AnswerKey == nil {
|
||||
t.Fatal("solution leaked or original mutated")
|
||||
}
|
||||
good := AIReview{SchemaVersion: 2, Approved: true, ChoiceChecks: []AIChoiceCheck{aiChoiceCheck(tasks[0], 0, key)}}
|
||||
if err := validateIndependentReview(tasks, good); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wrong := aiChoiceCheck(tasks[0], 0, tasks[0].Options[1].ID)
|
||||
ambiguous := aiChoiceCheck(tasks[0], 0, key, tasks[0].Options[1].ID)
|
||||
missing := aiChoiceCheck(tasks[0], 0, key)
|
||||
missing.Options = missing.Options[:1]
|
||||
duplicate := aiChoiceCheck(tasks[0], 0, key)
|
||||
duplicate.Options[1].OptionID = key
|
||||
outside := aiChoiceCheck(tasks[0], 1, key)
|
||||
for _, checks := range [][]AIChoiceCheck{nil, {wrong}, {ambiguous}, {missing}, {duplicate}, {outside}, {good.ChoiceChecks[0], good.ChoiceChecks[0]}} {
|
||||
bad := good
|
||||
bad.ChoiceChecks = checks
|
||||
if validateIndependentReview(tasks, bad) == nil {
|
||||
t.Fatal("unproven solution accepted", checks)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func setupAI(t *testing.T) (*AIService, *store.PostgresStore, string) {
|
||||
t.Helper()
|
||||
db, profiles := adaptiveTestDB(t)
|
||||
repo := store.NewAIStore(db)
|
||||
user, err := profiles.CreateUser(context.Background(), store.NewID("ai")+"@example.test", "hash", "de-DE")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := NewAIService(repo, config.Config{MistralAPIKey: "fixture-secret", DeepSeekAPIKey: "fixture-deepseek"})
|
||||
return a, profiles, user.ID
|
||||
}
|
||||
func TestAIPostgresSnapshotsUsageAndQuality(t *testing.T) {
|
||||
a, _, user := setupAI(t)
|
||||
ctx := context.Background()
|
||||
p := aiTestProfile("evaluate_course_attempt")
|
||||
if _, err := a.Repo.SaveProfile(ctx, user, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := a.Repo.SaveBudget(ctx, user, 1000000, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var count atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
count.Add(1)
|
||||
if r.Header.Get("Authorization") != "Bearer fixture-secret" {
|
||||
t.Error("missing credential")
|
||||
}
|
||||
var payload struct {
|
||||
Model string `json:"model"`
|
||||
Messages []struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
json.NewDecoder(r.Body).Decode(&payload)
|
||||
if payload.Model != "fixture-model" {
|
||||
t.Error("changed in-flight model")
|
||||
}
|
||||
if len(payload.Messages) != 2 || !strings.Contains(payload.Messages[0].Content, "Punktabzug") {
|
||||
t.Error("wrong system prompt")
|
||||
}
|
||||
var input struct {
|
||||
Task string `json:"task"`
|
||||
Input struct {
|
||||
Answer string `json:"answer"`
|
||||
} `json:"input"`
|
||||
}
|
||||
json.Unmarshal([]byte(payload.Messages[1].Content), &input)
|
||||
good := !strings.Contains(input.Input.Answer, "He have")
|
||||
evaluation := domain.CourseEvaluation{Score: 100, Verdict: "correct", Feedback: "Richtig", Issues: []string{}}
|
||||
if !good {
|
||||
evaluation = domain.CourseEvaluation{Score: 60, Verdict: "partial", Feedback: "He braucht has got.", Issues: []string{"Falsche Verbform have nach he"}}
|
||||
}
|
||||
json.NewEncoder(w).Encode(aiCompletion(evaluation, true))
|
||||
}))
|
||||
defer server.Close()
|
||||
a.URLs["mistral"] = server.URL
|
||||
ctx, o, err := a.Begin(ctx, user, "quality-operation", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Saving another profile while the operation runs must not change its model or price.
|
||||
p.Version = 2
|
||||
p.Model = "new-model"
|
||||
p.OutputPrice = 9000000
|
||||
if _, err = a.Repo.SaveProfile(ctx, user, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
results, err := a.Quality(ctx)
|
||||
if err != nil || len(results) != 2 {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range results {
|
||||
if !r.Passed {
|
||||
t.Fatalf("quality failed: %#v", r)
|
||||
}
|
||||
}
|
||||
if _, err = a.Quality(ctx); err != nil {
|
||||
t.Fatal("cached quality", err)
|
||||
}
|
||||
if count.Load() != 2 {
|
||||
t.Fatal("repeated request billed twice")
|
||||
}
|
||||
report, err := a.Repo.Usage(ctx, domain.AIUsageFilter{User: user})
|
||||
if err != nil || report.Total != 2 || report.InputTokens != 200 || report.OutputTokens != 40 || report.EstimatedCost != 208 {
|
||||
t.Fatalf("report %#v: %v", report, err)
|
||||
}
|
||||
for _, c := range report.Calls {
|
||||
if c.Profile.Version != 2 || c.Profile.Model != "fixture-model" || !c.Test || c.Input != "" {
|
||||
t.Fatalf("wrong frozen metadata: %#v", c)
|
||||
}
|
||||
detail, err := a.Repo.Call(ctx, c.ID)
|
||||
if err != nil || detail.Output == "" || strings.Contains(detail.Input, "fixture-secret") {
|
||||
t.Fatal("unsafe/missing detail", err)
|
||||
}
|
||||
}
|
||||
restarted, _, err := a.Begin(context.Background(), user, o.ID, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = a.Quality(restarted); err != nil || count.Load() != 2 {
|
||||
t.Fatal("restart ignored saved results", err)
|
||||
}
|
||||
}
|
||||
func TestAIPostgresConcurrentBudgetAndUnknown(t *testing.T) {
|
||||
a, _, user := setupAI(t)
|
||||
ctx := context.Background()
|
||||
repo := a.Repo
|
||||
if err := repo.SaveBudget(ctx, user, 100, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
success := atomic.Int32{}
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
id := store.NewID("budget")
|
||||
_, err := repo.Operation(ctx, domain.AIOperation{ID: id, UserID: user})
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
_, err = repo.Reserve(ctx, domain.AICall{OperationID: id, UserID: user, Action: "evaluate_course_attempt", Reserved: 60, Profile: aiTestProfile("evaluate_course_attempt")})
|
||||
if err == nil {
|
||||
success.Add(1)
|
||||
} else if !errors.Is(err, store.ErrAIBudget) {
|
||||
t.Error(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if success.Load() != 1 {
|
||||
t.Fatalf("budget raced: %d", success.Load())
|
||||
}
|
||||
budget, err := repo.Budget(ctx)
|
||||
if err != nil || budget.MonthCommitted != 60 || budget.OpenReservations != 60 {
|
||||
t.Fatal(budget, err)
|
||||
}
|
||||
report, _ := repo.Usage(ctx, domain.AIUsageFilter{})
|
||||
c := report.Calls[0]
|
||||
c.Status = "unknown"
|
||||
if err = repo.Finish(ctx, c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = repo.Resolve(ctx, user, c.ID, 10, "Provider invoice reconciled"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
budget, _ = repo.Budget(ctx)
|
||||
if budget.MonthCommitted != 10 || budget.OpenReservations != 0 {
|
||||
t.Fatal("reservation released incorrectly", budget)
|
||||
}
|
||||
c, _ = repo.Call(ctx, c.ID)
|
||||
if !strings.Contains(c.Error, user) {
|
||||
t.Fatal("reconciliation unaudited")
|
||||
}
|
||||
}
|
||||
func TestAIPostgresProviderFailuresAndUnknownUsage(t *testing.T) {
|
||||
for _, scenario := range []string{"missing_usage", "length", "unauthorized", "rate_limit", "disconnect"} {
|
||||
t.Run(scenario, func(t *testing.T) {
|
||||
a, _, user := setupAI(t)
|
||||
ctx := context.Background()
|
||||
p := aiTestProfile("evaluate_course_attempt")
|
||||
p.Provider = "deepseek"
|
||||
p.ReasoningEffort = "none"
|
||||
if _, err := a.Repo.SaveProfile(ctx, user, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.Repo.SaveBudget(ctx, user, 1000000, 0)
|
||||
var calls atomic.Int32
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
calls.Add(1)
|
||||
var payload map[string]any
|
||||
json.NewDecoder(r.Body).Decode(&payload)
|
||||
if _, ok := payload["reasoning_effort"]; ok {
|
||||
t.Error("none effort leaked to DeepSeek")
|
||||
}
|
||||
if payload["thinking"].(map[string]any)["type"] != "disabled" {
|
||||
t.Error("thinking not disabled")
|
||||
}
|
||||
if scenario == "unauthorized" {
|
||||
w.WriteHeader(401)
|
||||
io.WriteString(w, `{"error":{"message":"Invalid fixture-secret"}}`)
|
||||
return
|
||||
}
|
||||
if scenario == "rate_limit" {
|
||||
w.WriteHeader(429)
|
||||
io.WriteString(w, `{"message":"Rate limit exceeded fixture-secret"}`)
|
||||
return
|
||||
}
|
||||
if scenario == "disconnect" {
|
||||
conn, _, _ := w.(http.Hijacker).Hijack()
|
||||
conn.Close()
|
||||
return
|
||||
}
|
||||
result := aiCompletion(domain.CourseEvaluation{Score: 100, Verdict: "correct", Feedback: "Richtig", Issues: []string{}}, scenario != "missing_usage")
|
||||
if scenario == "length" {
|
||||
result["choices"] = []any{map[string]any{"finish_reason": "length", "message": map[string]any{"content": "{"}}}
|
||||
}
|
||||
json.NewEncoder(w).Encode(result)
|
||||
}))
|
||||
defer server.Close()
|
||||
a.URLs["deepseek"] = server.URL
|
||||
ctx, _, err := a.Begin(ctx, user, store.NewID("failure"), true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var evaluation domain.CourseEvaluation
|
||||
err = a.Call(ctx, "evaluate_course_attempt", map[string]any{"answer": "He has got a sister."}, &evaluation)
|
||||
report, _ := a.Repo.Usage(ctx, domain.AIUsageFilter{})
|
||||
switch scenario {
|
||||
case "missing_usage":
|
||||
if err != nil || report.Unknown != 1 || report.Reserved == 0 {
|
||||
t.Fatal("missing usage treated as zero", err, report)
|
||||
}
|
||||
case "length":
|
||||
if err == nil || calls.Load() != 2 || report.Total != 2 || report.EstimatedCost == 0 {
|
||||
t.Fatal("invalid completion not charged or unbounded retries", err, report)
|
||||
}
|
||||
case "unauthorized":
|
||||
if err == nil || calls.Load() != 1 || report.Reserved != 0 || report.EstimatedCost != 0 {
|
||||
t.Fatal("rejected request billing", err, report)
|
||||
}
|
||||
if strings.Contains(err.Error(), "fixture-secret") || !strings.Contains(err.Error(), "Invalid [redacted]") {
|
||||
t.Fatal("provider message missing or credential exposed", err)
|
||||
}
|
||||
case "rate_limit":
|
||||
if err == nil || calls.Load() != 1 || report.Reserved != 0 || report.EstimatedCost != 0 || !strings.Contains(err.Error(), "HTTP 429 : Rate limit exceeded [redacted]") || strings.Contains(err.Error(), "fixture-secret") {
|
||||
t.Fatal("rate limit reason or zero billing lost", err, report)
|
||||
}
|
||||
case "disconnect":
|
||||
if err == nil || calls.Load() != 1 || report.Reserved == 0 {
|
||||
t.Fatal("uncertain bill lost", err, report)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
func TestAIPostgresDirectCoursePipeline(t *testing.T) {
|
||||
db, profiles := adaptiveTestDB(t)
|
||||
account, err := profiles.CreateUser(context.Background(), "direct-course@example.test", "hash", "de-DE")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
user := account.ID
|
||||
a := NewAIService(store.NewAIStore(db), config.Config{MistralAPIKey: "fixture-secret"})
|
||||
ctx := context.Background()
|
||||
profiles.CreateLearnerProfile(ctx, user, "en-US", "A1", 10)
|
||||
for _, action := range []string{"analyze_course_progress", "plan_course_task", "author_course_task", "review_course_task", "next_exercise"} {
|
||||
if _, err := a.Repo.SaveProfile(ctx, user, aiTestProfile(action)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
a.Repo.SaveBudget(ctx, user, 10000000, 0)
|
||||
var authors atomic.Int32
|
||||
var candidate any
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var body struct {
|
||||
Messages []struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"messages"`
|
||||
}
|
||||
json.NewDecoder(r.Body).Decode(&body)
|
||||
var q struct {
|
||||
Task string `json:"task"`
|
||||
Input json.RawMessage `json:"input"`
|
||||
}
|
||||
json.Unmarshal([]byte(body.Messages[1].Content), &q)
|
||||
var result any
|
||||
switch q.Task {
|
||||
case "analyze_course_progress":
|
||||
mods := []domain.ModuleAnalysis{}
|
||||
for _, m := range store.EnglishCoursePack().Modules {
|
||||
mods = append(mods, domain.ModuleAnalysis{ModuleID: m.ID, Strengths: []string{}, Weaknesses: []string{}, Recommendation: "Bereich ausprobieren", Reason: "Noch keine Nachweise", EvidenceIDs: []string{}})
|
||||
}
|
||||
result = domain.CourseAnalysis{SchemaVersion: 2, TargetLanguage: "en-US", CourseVersion: domain.CourseVersion, Summary: "Noch keine Nachweise.", RecommendedModuleID: "a1_introductions_v1", Modules: mods}
|
||||
case "plan_course_task":
|
||||
result = domain.CourseBrief{SchemaVersion: 2, ModuleID: "a1_family_v1", TargetLanguage: "en-US", CourseVersion: domain.CourseVersion, Skill: "grammar", Kind: "choice", Objective: "have und has unterscheiden", Difficulty: "A1", Reason: "Einstieg"}
|
||||
case "author_course_task":
|
||||
authors.Add(1)
|
||||
task := store.EnglishCoursePack().Reserves[7]
|
||||
task.ID = ""
|
||||
task.Source = ""
|
||||
candidate = map[string]any{"schema_version": 2, "tasks": []domain.AdaptiveTask{task}}
|
||||
result = candidate
|
||||
case "review_course_task":
|
||||
var input struct {
|
||||
Candidate struct {
|
||||
Tasks []domain.AdaptiveTask `json:"tasks"`
|
||||
} `json:"candidate"`
|
||||
}
|
||||
json.Unmarshal(q.Input, &input)
|
||||
if len(input.Candidate.Tasks) != 1 {
|
||||
t.Error("reviewer missing author output")
|
||||
}
|
||||
if input.Candidate.Tasks[0].AnswerKey != nil {
|
||||
t.Error("reviewer was shown the authored solution")
|
||||
}
|
||||
result = AIReview{SchemaVersion: 2, Approved: true, ChoiceChecks: []AIChoiceCheck{aiChoiceCheck(store.EnglishCoursePack().Reserves[7], 0, store.EnglishCoursePack().Reserves[7].AnswerKey.CorrectOptionID)}, ReviewNotes: "Unabhängig gelöst"}
|
||||
case "next_exercise":
|
||||
result = map[string]any{"exercise_type": "repeat_sentence", "target_language": "en-US", "cefr_level": "A1", "reference_text": "My sister is here.", "display_prompt": "Sprich nach.", "tts_text": "My sister is here.", "focus_points": []string{"sister"}, "success_criteria": []string{"verständlich sprechen"}, "reason_for_choice": "Familienwort"}
|
||||
default:
|
||||
t.Errorf("unknown operation %s", q.Task)
|
||||
}
|
||||
json.NewEncoder(w).Encode(aiCompletion(result, true))
|
||||
}))
|
||||
defer server.Close()
|
||||
a.URLs["mistral"] = server.URL
|
||||
engine := NewCourseEngine(store.NewAdaptiveStore(db), profiles, a, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
job, err := engine.Repo.Enqueue(ctx, user, "en-US", "next", "a1_family_v1", "grammar", "", "direct-request")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed, err := engine.Repo.Claim(ctx)
|
||||
if err != nil || claimed.ID != job.ID {
|
||||
t.Fatal(err)
|
||||
}
|
||||
generated, err := engine.Process(ctx, claimed)
|
||||
if err != nil || generated.Source != "ai" || authors.Load() != 1 {
|
||||
t.Fatal(generated, err)
|
||||
}
|
||||
stored, err := engine.Repo.Task(ctx, user, "en-US", generated.ID)
|
||||
if err != nil || stored.Source != "ai" || stored.AnswerKey == nil {
|
||||
t.Fatal("course provenance/storage", stored, err)
|
||||
}
|
||||
if err = engine.Repo.Finish(ctx, claimed, generated, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exercise, err := a.NextExercise(ctx, domain.GeneratedExerciseRequest{UserID: user, TargetLanguage: "en-US", CEFRLevel: "A1"})
|
||||
if err != nil || exercise.Source != "ai" {
|
||||
t.Fatal(exercise, err)
|
||||
}
|
||||
if _, saved, err := profiles.SaveGeneratedExercise(ctx, user, exercise); err != nil || saved.Source != "ai" {
|
||||
t.Fatal("pronunciation persistence", saved, err)
|
||||
}
|
||||
}
|
||||
|
||||
// A response timeout is intentionally not retried blindly: billing may have
|
||||
// occurred, and the remaining budget reservation must remain visible.
|
||||
func TestAICallDeadlineWithoutNetwork(t *testing.T) {
|
||||
a, _, user := setupAI(t)
|
||||
ctx := context.Background()
|
||||
a.Repo.SaveProfile(ctx, user, aiTestProfile("evaluate_course_attempt"))
|
||||
a.Repo.SaveBudget(ctx, user, 1000000, 0)
|
||||
a.Client = &http.Client{Transport: speechRoundTripFunc(func(r *http.Request) (*http.Response, error) { return nil, context.DeadlineExceeded })}
|
||||
work, _, err := a.Begin(ctx, user, "deadline", true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out domain.CourseEvaluation
|
||||
start := time.Now()
|
||||
if err = a.Call(work, "evaluate_course_attempt", map[string]string{"answer": "Hello"}, &out); err == nil {
|
||||
t.Fatal("timeout ignored")
|
||||
}
|
||||
if time.Since(start) > time.Second {
|
||||
t.Fatal("unexpected wait")
|
||||
}
|
||||
report, _ := a.Repo.Usage(ctx, domain.AIUsageFilter{})
|
||||
if report.Total != 1 || report.Unknown != 1 || report.Reserved == 0 {
|
||||
t.Fatal(report)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAIPostgresDailyLimitAndUnknownResolution(t *testing.T) {
|
||||
a, profiles, user := setupAI(t)
|
||||
ctx := context.Background()
|
||||
repo := a.Repo
|
||||
p := aiTestProfile("evaluate_course_attempt")
|
||||
saved, err := repo.SaveProfile(ctx, user, p)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = repo.SaveProfile(ctx, user, p); !errors.Is(err, store.ErrAIConflict) {
|
||||
t.Fatal("stale settings overwrote profile", err)
|
||||
}
|
||||
if err = profiles.Init(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
all, err := repo.Profiles(ctx)
|
||||
if err != nil || all[p.Action].Version != saved.Version || !all[p.Action].Enabled {
|
||||
t.Fatal("migration reset settings", err)
|
||||
}
|
||||
if err = repo.SaveBudget(ctx, user, 1000000, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reserve := func(id string) (domain.AICall, error) {
|
||||
_, err := repo.Operation(ctx, domain.AIOperation{ID: id, UserID: user})
|
||||
if err != nil {
|
||||
return domain.AICall{}, err
|
||||
}
|
||||
return repo.Reserve(ctx, domain.AICall{OperationID: id, UserID: user, Action: p.Action, Profile: saved, Reserved: 60})
|
||||
}
|
||||
c, err := reserve("daily-one")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = reserve("daily-two"); !errors.Is(err, store.ErrAIBudget) {
|
||||
t.Fatal("daily limit ignored", err)
|
||||
}
|
||||
c.Status = "succeeded"
|
||||
c.Output = `{"score":100,"verdict":"correct","feedback":"Gut","issues":[]}`
|
||||
if err = repo.Finish(ctx, c); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Successful output with missing usage remains cached and can be reconciled.
|
||||
if err = repo.Resolve(ctx, user, c.ID, 10, "Abgleich mit Anbieterrechnung"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = repo.Resolve(ctx, user, c.ID, 0, "Erneuter Abgleich"); !errors.Is(err, store.ErrAIConflict) {
|
||||
t.Fatal("reconciliation overwritten", err)
|
||||
}
|
||||
cached, err := repo.Cached(ctx, c.OperationID, c.Action, c.InputHash)
|
||||
if err != nil || cached == nil || cached.Cost == nil || *cached.Cost != 10 {
|
||||
t.Fatal("reconciliation lost cached output", err)
|
||||
}
|
||||
if _, err = reserve("daily-two"); err != nil {
|
||||
t.Fatal("released budget unavailable", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAIPostgresExplicitRetryRefreshesFrozenProfile(t *testing.T) {
|
||||
db, profiles := adaptiveTestDB(t)
|
||||
ctx := context.Background()
|
||||
account, err := profiles.CreateUser(ctx, "retry-ai@example.test", "hash", "de-DE")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := NewAIService(store.NewAIStore(db), config.Config{MistralAPIKey: "fixture-secret"})
|
||||
engine := NewCourseEngine(store.NewAdaptiveStore(db), profiles, a, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
||||
task, err := engine.Repo.Reserve(ctx, account.ID, "en-US", "a1_family_v1", "writing")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
attempt, job, err := engine.Submit(ctx, account.ID, "en-US", task.ID, "retry-saved-answer", "He has got a sister.", nil)
|
||||
if err != nil || job == nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
claimed, err := engine.Repo.Claim(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = engine.Process(ctx, claimed); !errors.Is(err, store.ErrAIDisabled) {
|
||||
t.Fatal("disabled profile not detected", err)
|
||||
}
|
||||
engine.Repo.FailAttempt(ctx, account.ID, attempt.ID)
|
||||
engine.Repo.Finish(ctx, claimed, nil, "disabled")
|
||||
p := aiTestProfile("evaluate_course_attempt")
|
||||
if _, err = a.Repo.SaveProfile(ctx, account.ID, p); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.Repo.SaveBudget(ctx, account.ID, 1000000, 0)
|
||||
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
json.NewEncoder(w).Encode(aiCompletion(domain.CourseEvaluation{Score: 100, Verdict: "correct", Feedback: "Richtig", Issues: []string{}}, true))
|
||||
}))
|
||||
defer provider.Close()
|
||||
a.URLs["mistral"] = provider.URL
|
||||
retried, err := engine.Repo.RetryEvaluation(ctx, account.ID, "en-US", attempt.ID)
|
||||
if err != nil || retried.AIRevision != 1 {
|
||||
t.Fatal("new retry revision missing", retried, err)
|
||||
}
|
||||
claimed, err = engine.Repo.Claim(ctx)
|
||||
if err != nil || claimed.ID != job.ID {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = engine.Process(ctx, claimed); err != nil {
|
||||
t.Fatal("retry retained unusable old settings", err)
|
||||
}
|
||||
preserved, err := engine.Repo.Attempt(ctx, account.ID, "en-US", attempt.ID)
|
||||
if err != nil || preserved.Answer != "He has got a sister." || preserved.Evaluation == nil || preserved.Evaluation.Score != 100 {
|
||||
t.Fatal("answer lost during retry", preserved, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAIPostgresOperationDoesNotMutateSharedPrompts(t *testing.T) {
|
||||
a, _, user := setupAI(t)
|
||||
ctx := context.Background()
|
||||
action := "evaluate_course_attempt"
|
||||
old := a.Prompts[action]
|
||||
if _, _, err := a.Begin(ctx, user, "frozen-prompts", true); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a.Prompts[action] = "new prompt version"
|
||||
a.PromptVersion = "new-version"
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 8; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
_, frozen, err := a.Begin(ctx, user, "frozen-prompts", true)
|
||||
if err != nil || frozen.Prompts[action] != old {
|
||||
t.Error("old prompt snapshot lost", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
if a.Prompts[action] != "new prompt version" {
|
||||
t.Fatal("reading old operation overwrote shared application prompts")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user