Direkte DeepSeek- und Mistral-Anbindung mit versionierten Aktionsprofilen, Budgetreservierung, Tokenabrechnung und administrativen Qualitätsprüfungen ersetzen die Flowise-Abhängigkeit im regulären Betrieb. 840 geprüfte A1-Aufgaben mit geschützter Testauswertung und konkreten offenen Arbeitsaufträgen ergänzen den Kurs. Inhalts- und Sprachbewertung bleiben von Azure-Aussprachewerten getrennt; historische Aufgaben und Antworten bleiben erhalten. Echte Wort- und Phonemmessungen werden mit Herkunft, Aktualität und Nachweisdichte gebündelt. Go bestimmt Trainingsziele und Wiederholungen, das Modell formuliert dazu passende Nachsprechübungen. Profilansicht und Auswahlgründe werden angezeigt. Validierung: Go-Race-Tests mit lokaler PostgreSQL-Integration, go vet, Web-Build, Audit aller 840 Aufgaben und drei Browser-Smokes mit lokalen Modell-/Speech-Fixtures bestanden. Keine Produktionsänderungen oder Live-Aufrufe.
673 lines
25 KiB
Go
673 lines
25 KiB
Go
package services
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"speech-learning-api/internal/config"
|
|
"speech-learning-api/internal/domain"
|
|
"speech-learning-api/internal/store"
|
|
)
|
|
|
|
func aiTestProfile(action string) domain.AIProfile {
|
|
return domain.AIProfile{Action: action, Version: 1, Enabled: true, Provider: "mistral", Model: "fixture-model", MaxTokens: 2048, ContextTokens: 32768, TimeoutSeconds: 5, PricingSet: true, InputPrice: 1000000, CachedInputPrice: 100000, OutputPrice: 2000000}
|
|
}
|
|
func aiCompletion(content any, usage bool) map[string]any {
|
|
b, _ := json.Marshal(content)
|
|
r := map[string]any{"choices": []any{map[string]any{"finish_reason": "stop", "message": map[string]any{"content": string(b)}}}}
|
|
if usage {
|
|
r["usage"] = map[string]any{"prompt_tokens": 100, "completion_tokens": 20, "prompt_cache_hit_tokens": 40, "completion_tokens_details": map[string]any{"reasoning_tokens": 5}}
|
|
}
|
|
return r
|
|
}
|
|
func TestAICompletionAccountingAndContracts(t *testing.T) {
|
|
good := domain.CourseEvaluation{Score: 100, Verdict: "correct", Feedback: "Richtig", Issues: []string{}}
|
|
c := domain.AICall{Action: "evaluate_course_attempt", Profile: aiTestProfile("evaluate_course_attempt")}
|
|
b, _ := json.Marshal(aiCompletion(good, true))
|
|
var out domain.CourseEvaluation
|
|
if err := readAICompletion(b, &c, &out); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Reasoning is a subset of completion tokens; cache hits are a subset of input.
|
|
if c.Cost == nil || *c.Cost != 104 || *c.Usage.ReasoningTokens != 5 || c.Usage.OutputTokens != 20 {
|
|
t.Fatalf("double counting: %#v", c)
|
|
}
|
|
c = domain.AICall{Action: "evaluate_course_attempt", Profile: aiTestProfile("evaluate_course_attempt")}
|
|
b, _ = json.Marshal(aiCompletion(good, false))
|
|
if err := readAICompletion(b, &c, &out); err != nil || c.Cost != nil || c.Usage != nil {
|
|
t.Fatal("missing usage fabricated", err)
|
|
}
|
|
for _, content := range []any{map[string]any{"score": 100}, map[string]any{"verdict": "correct", "feedback": "Ja", "issues": []string{}}, map[string]any{"score": 100, "verdict": "correct", "feedback": "Ja", "issues": []string{}, "owner_id": "evil"}} {
|
|
c = domain.AICall{Action: "evaluate_course_attempt", Profile: aiTestProfile("evaluate_course_attempt")}
|
|
b, _ = json.Marshal(aiCompletion(content, true))
|
|
if readAICompletion(b, &c, &out) == nil {
|
|
t.Fatal("incomplete contract accepted")
|
|
}
|
|
if c.Cost == nil {
|
|
t.Fatal("invalid JSON lost billable usage")
|
|
}
|
|
}
|
|
raw := aiCompletion(good, true)
|
|
raw["choices"] = []any{map[string]any{"finish_reason": "length", "message": map[string]any{"content": "{\"score\":"}}}
|
|
b, _ = json.Marshal(raw)
|
|
c = domain.AICall{Profile: aiTestProfile("evaluate_course_attempt")}
|
|
if readAICompletion(b, &c, &out) == nil || c.Cost == nil {
|
|
t.Fatal("truncated reply or cost lost")
|
|
}
|
|
}
|
|
func TestAIProfilesAndPromptIsolation(t *testing.T) {
|
|
a := NewAIService(nil, config.Config{})
|
|
if len(a.Prompts) != len(domain.AIActions) || a.PromptVersion == "" {
|
|
t.Fatal("missing embedded prompts")
|
|
}
|
|
eval := a.Prompts["evaluate_course_attempt"]
|
|
if strings.Contains(eval, "Für task=\"plan_course_task\"") {
|
|
t.Fatal("mixed task prompt")
|
|
}
|
|
if !strings.Contains(eval, "Punktabzug") || !strings.Contains(a.Prompts["analyze_course_progress"], "distinct_tasks") {
|
|
t.Fatal("quality regressions missing")
|
|
}
|
|
p := aiTestProfile("evaluate_course_attempt")
|
|
if err := ValidAIProfile(p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p.PricingSet = false
|
|
if ValidAIProfile(p) == nil {
|
|
t.Fatal("unpriced enabled model")
|
|
}
|
|
p.PricingSet = true
|
|
p.Provider = "deepseek"
|
|
p.JSONSchema = true
|
|
if ValidAIProfile(p) == nil {
|
|
t.Fatal("unsupported DeepSeek schema mode")
|
|
}
|
|
}
|
|
|
|
func TestMistralTaskSchemasKeepRendererRequirements(t *testing.T) {
|
|
a := NewAIService(nil, config.Config{})
|
|
for _, test := range []struct{ skill, kind, required, absent string }{
|
|
{"grammar", "choice", "options", "audio_text"},
|
|
{"listening", "choice", "audio_text", "passage"},
|
|
{"reading", "short_text", "passage", "options"},
|
|
{"speaking", "repeat_sentence", "reference_text", "options"},
|
|
{"speaking", "free_speech", "answer_key", "reference_text"},
|
|
{"writing", "free_text", "answer_key", "options"},
|
|
} {
|
|
t.Run(test.skill+"/"+test.kind, func(t *testing.T) {
|
|
var full map[string]any
|
|
if err := json.Unmarshal([]byte(a.Schemas["author_course_task"]), &full); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
before, _ := json.Marshal(full)
|
|
schema := mistralStructuralSchema(full)
|
|
constrainTaskSchema(schema, "author_course_task", map[string]any{"brief": map[string]any{"module_id": "a1_family_v1", "skill": test.skill, "kind": test.kind}})
|
|
after, _ := json.Marshal(full)
|
|
if string(before) != string(after) {
|
|
t.Fatal("persisted contract was mutated")
|
|
}
|
|
props := schema.(map[string]any)["properties"].(map[string]any)
|
|
tasks := props["tasks"].(map[string]any)
|
|
item := tasks["items"].(map[string]any)
|
|
fields := item["properties"].(map[string]any)
|
|
if tasks["minItems"] != 1 || tasks["maxItems"] != 1 || item["allOf"] != nil || fields[test.absent] != nil {
|
|
t.Fatal("wrong decoder contract", tasks)
|
|
}
|
|
found := false
|
|
for _, field := range item["required"].([]any) {
|
|
found = found || field == test.required
|
|
}
|
|
if !found || fields["kind"].(map[string]any)["const"] != test.kind {
|
|
t.Fatal("required renderer fields lost")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func aiChoiceCheck(task domain.AdaptiveTask, index int, validIDs ...string) AIChoiceCheck {
|
|
check := AIChoiceCheck{TaskIndex: index}
|
|
for _, option := range task.Options {
|
|
valid := false
|
|
for _, id := range validIDs {
|
|
valid = valid || id == option.ID
|
|
}
|
|
check.Options = append(check.Options, AIOptionCheck{OptionID: option.ID, IsCorrect: valid, Reason: "Test reasoning"})
|
|
}
|
|
return check
|
|
}
|
|
func TestIndependentReviewRejectsAnchoredAndAmbiguousSolutions(t *testing.T) {
|
|
tasks := []domain.AdaptiveTask{store.EnglishCoursePack().Reserves[7]}
|
|
key := tasks[0].AnswerKey.CorrectOptionID
|
|
blind := blindReviewTasks(tasks)
|
|
if blind[0].AnswerKey != nil || tasks[0].AnswerKey == nil {
|
|
t.Fatal("solution leaked or original mutated")
|
|
}
|
|
good := AIReview{SchemaVersion: 2, Approved: true, ChoiceChecks: []AIChoiceCheck{aiChoiceCheck(tasks[0], 0, key)}}
|
|
if err := validateIndependentReview(tasks, good); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wrong := aiChoiceCheck(tasks[0], 0, tasks[0].Options[1].ID)
|
|
ambiguous := aiChoiceCheck(tasks[0], 0, key, tasks[0].Options[1].ID)
|
|
missing := aiChoiceCheck(tasks[0], 0, key)
|
|
missing.Options = missing.Options[:1]
|
|
duplicate := aiChoiceCheck(tasks[0], 0, key)
|
|
duplicate.Options[1].OptionID = key
|
|
outside := aiChoiceCheck(tasks[0], 1, key)
|
|
for _, checks := range [][]AIChoiceCheck{nil, {wrong}, {ambiguous}, {missing}, {duplicate}, {outside}, {good.ChoiceChecks[0], good.ChoiceChecks[0]}} {
|
|
bad := good
|
|
bad.ChoiceChecks = checks
|
|
if validateIndependentReview(tasks, bad) == nil {
|
|
t.Fatal("unproven solution accepted", checks)
|
|
}
|
|
}
|
|
}
|
|
|
|
func setupAI(t *testing.T) (*AIService, *store.PostgresStore, string) {
|
|
t.Helper()
|
|
db, profiles := adaptiveTestDB(t)
|
|
repo := store.NewAIStore(db)
|
|
user, err := profiles.CreateUser(context.Background(), store.NewID("ai")+"@example.test", "hash", "de-DE")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a := NewAIService(repo, config.Config{MistralAPIKey: "fixture-secret", DeepSeekAPIKey: "fixture-deepseek"})
|
|
return a, profiles, user.ID
|
|
}
|
|
func TestAIPostgresSnapshotsUsageAndQuality(t *testing.T) {
|
|
a, _, user := setupAI(t)
|
|
ctx := context.Background()
|
|
p := aiTestProfile("evaluate_course_attempt")
|
|
if _, err := a.Repo.SaveProfile(ctx, user, p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := a.Repo.SaveBudget(ctx, user, 1000000, 0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var count atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
count.Add(1)
|
|
if r.Header.Get("Authorization") != "Bearer fixture-secret" {
|
|
t.Error("missing credential")
|
|
}
|
|
var payload struct {
|
|
Model string `json:"model"`
|
|
Messages []struct {
|
|
Role string `json:"role"`
|
|
Content string `json:"content"`
|
|
} `json:"messages"`
|
|
}
|
|
json.NewDecoder(r.Body).Decode(&payload)
|
|
if payload.Model != "fixture-model" {
|
|
t.Error("changed in-flight model")
|
|
}
|
|
if len(payload.Messages) != 2 || !strings.Contains(payload.Messages[0].Content, "Punktabzug") {
|
|
t.Error("wrong system prompt")
|
|
}
|
|
var input struct {
|
|
Task string `json:"task"`
|
|
Input struct {
|
|
Answer string `json:"answer"`
|
|
} `json:"input"`
|
|
}
|
|
json.Unmarshal([]byte(payload.Messages[1].Content), &input)
|
|
good := !strings.Contains(input.Input.Answer, "He have")
|
|
evaluation := domain.CourseEvaluation{Score: 100, Verdict: "correct", Feedback: "Richtig", Issues: []string{}}
|
|
if !good {
|
|
evaluation = domain.CourseEvaluation{Score: 60, Verdict: "partial", Feedback: "He braucht has got.", Issues: []string{"Falsche Verbform have nach he"}}
|
|
}
|
|
json.NewEncoder(w).Encode(aiCompletion(evaluation, true))
|
|
}))
|
|
defer server.Close()
|
|
a.URLs["mistral"] = server.URL
|
|
ctx, o, err := a.Begin(ctx, user, "quality-operation", true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Saving another profile while the operation runs must not change its model or price.
|
|
p.Version = 2
|
|
p.Model = "new-model"
|
|
p.OutputPrice = 9000000
|
|
if _, err = a.Repo.SaveProfile(ctx, user, p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
results, err := a.Quality(ctx)
|
|
if err != nil || len(results) != 2 {
|
|
t.Fatal(err)
|
|
}
|
|
for _, r := range results {
|
|
if !r.Passed {
|
|
t.Fatalf("quality failed: %#v", r)
|
|
}
|
|
}
|
|
if _, err = a.Quality(ctx); err != nil {
|
|
t.Fatal("cached quality", err)
|
|
}
|
|
if count.Load() != 2 {
|
|
t.Fatal("repeated request billed twice")
|
|
}
|
|
report, err := a.Repo.Usage(ctx, domain.AIUsageFilter{User: user})
|
|
if err != nil || report.Total != 2 || report.InputTokens != 200 || report.OutputTokens != 40 || report.EstimatedCost != 208 {
|
|
t.Fatalf("report %#v: %v", report, err)
|
|
}
|
|
for _, c := range report.Calls {
|
|
if c.Profile.Version != 2 || c.Profile.Model != "fixture-model" || !c.Test || c.Input != "" {
|
|
t.Fatalf("wrong frozen metadata: %#v", c)
|
|
}
|
|
detail, err := a.Repo.Call(ctx, c.ID)
|
|
if err != nil || detail.Output == "" || strings.Contains(detail.Input, "fixture-secret") {
|
|
t.Fatal("unsafe/missing detail", err)
|
|
}
|
|
}
|
|
restarted, _, err := a.Begin(context.Background(), user, o.ID, true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = a.Quality(restarted); err != nil || count.Load() != 2 {
|
|
t.Fatal("restart ignored saved results", err)
|
|
}
|
|
}
|
|
func TestAIPostgresConcurrentBudgetAndUnknown(t *testing.T) {
|
|
a, _, user := setupAI(t)
|
|
ctx := context.Background()
|
|
repo := a.Repo
|
|
if err := repo.SaveBudget(ctx, user, 100, 0); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
success := atomic.Int32{}
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 8; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
id := store.NewID("budget")
|
|
_, err := repo.Operation(ctx, domain.AIOperation{ID: id, UserID: user})
|
|
if err != nil {
|
|
t.Error(err)
|
|
return
|
|
}
|
|
_, err = repo.Reserve(ctx, domain.AICall{OperationID: id, UserID: user, Action: "evaluate_course_attempt", Reserved: 60, Profile: aiTestProfile("evaluate_course_attempt")})
|
|
if err == nil {
|
|
success.Add(1)
|
|
} else if !errors.Is(err, store.ErrAIBudget) {
|
|
t.Error(err)
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
if success.Load() != 1 {
|
|
t.Fatalf("budget raced: %d", success.Load())
|
|
}
|
|
budget, err := repo.Budget(ctx)
|
|
if err != nil || budget.MonthCommitted != 60 || budget.OpenReservations != 60 {
|
|
t.Fatal(budget, err)
|
|
}
|
|
report, _ := repo.Usage(ctx, domain.AIUsageFilter{})
|
|
c := report.Calls[0]
|
|
c.Status = "unknown"
|
|
if err = repo.Finish(ctx, c); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = repo.Resolve(ctx, user, c.ID, 10, "Provider invoice reconciled"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
budget, _ = repo.Budget(ctx)
|
|
if budget.MonthCommitted != 10 || budget.OpenReservations != 0 {
|
|
t.Fatal("reservation released incorrectly", budget)
|
|
}
|
|
c, _ = repo.Call(ctx, c.ID)
|
|
if !strings.Contains(c.Error, user) {
|
|
t.Fatal("reconciliation unaudited")
|
|
}
|
|
}
|
|
func TestAIPostgresProviderFailuresAndUnknownUsage(t *testing.T) {
|
|
for _, scenario := range []string{"missing_usage", "length", "unauthorized", "rate_limit", "disconnect"} {
|
|
t.Run(scenario, func(t *testing.T) {
|
|
a, _, user := setupAI(t)
|
|
ctx := context.Background()
|
|
p := aiTestProfile("evaluate_course_attempt")
|
|
p.Provider = "deepseek"
|
|
p.ReasoningEffort = "none"
|
|
if _, err := a.Repo.SaveProfile(ctx, user, p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a.Repo.SaveBudget(ctx, user, 1000000, 0)
|
|
var calls atomic.Int32
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
calls.Add(1)
|
|
var payload map[string]any
|
|
json.NewDecoder(r.Body).Decode(&payload)
|
|
if _, ok := payload["reasoning_effort"]; ok {
|
|
t.Error("none effort leaked to DeepSeek")
|
|
}
|
|
if payload["thinking"].(map[string]any)["type"] != "disabled" {
|
|
t.Error("thinking not disabled")
|
|
}
|
|
if scenario == "unauthorized" {
|
|
w.WriteHeader(401)
|
|
io.WriteString(w, `{"error":{"message":"Invalid fixture-secret"}}`)
|
|
return
|
|
}
|
|
if scenario == "rate_limit" {
|
|
w.WriteHeader(429)
|
|
io.WriteString(w, `{"message":"Rate limit exceeded fixture-secret"}`)
|
|
return
|
|
}
|
|
if scenario == "disconnect" {
|
|
conn, _, _ := w.(http.Hijacker).Hijack()
|
|
conn.Close()
|
|
return
|
|
}
|
|
result := aiCompletion(domain.CourseEvaluation{Score: 100, Verdict: "correct", Feedback: "Richtig", Issues: []string{}}, scenario != "missing_usage")
|
|
if scenario == "length" {
|
|
result["choices"] = []any{map[string]any{"finish_reason": "length", "message": map[string]any{"content": "{"}}}
|
|
}
|
|
json.NewEncoder(w).Encode(result)
|
|
}))
|
|
defer server.Close()
|
|
a.URLs["deepseek"] = server.URL
|
|
ctx, _, err := a.Begin(ctx, user, store.NewID("failure"), true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var evaluation domain.CourseEvaluation
|
|
err = a.Call(ctx, "evaluate_course_attempt", map[string]any{"answer": "He has got a sister."}, &evaluation)
|
|
report, _ := a.Repo.Usage(ctx, domain.AIUsageFilter{})
|
|
switch scenario {
|
|
case "missing_usage":
|
|
if err != nil || report.Unknown != 1 || report.Reserved == 0 {
|
|
t.Fatal("missing usage treated as zero", err, report)
|
|
}
|
|
case "length":
|
|
if err == nil || calls.Load() != 2 || report.Total != 2 || report.EstimatedCost == 0 {
|
|
t.Fatal("invalid completion not charged or unbounded retries", err, report)
|
|
}
|
|
case "unauthorized":
|
|
if err == nil || calls.Load() != 1 || report.Reserved != 0 || report.EstimatedCost != 0 {
|
|
t.Fatal("rejected request billing", err, report)
|
|
}
|
|
if strings.Contains(err.Error(), "fixture-secret") || !strings.Contains(err.Error(), "Invalid [redacted]") {
|
|
t.Fatal("provider message missing or credential exposed", err)
|
|
}
|
|
case "rate_limit":
|
|
if err == nil || calls.Load() != 1 || report.Reserved != 0 || report.EstimatedCost != 0 || !strings.Contains(err.Error(), "HTTP 429 : Rate limit exceeded [redacted]") || strings.Contains(err.Error(), "fixture-secret") {
|
|
t.Fatal("rate limit reason or zero billing lost", err, report)
|
|
}
|
|
case "disconnect":
|
|
if err == nil || calls.Load() != 1 || report.Reserved == 0 {
|
|
t.Fatal("uncertain bill lost", err, report)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
func TestAIPostgresDirectCoursePipeline(t *testing.T) {
|
|
db, profiles := adaptiveTestDB(t)
|
|
account, err := profiles.CreateUser(context.Background(), "direct-course@example.test", "hash", "de-DE")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
user := account.ID
|
|
a := NewAIService(store.NewAIStore(db), config.Config{MistralAPIKey: "fixture-secret"})
|
|
ctx := context.Background()
|
|
profiles.CreateLearnerProfile(ctx, user, "en-US", "A1", 10)
|
|
for _, action := range []string{"analyze_course_progress", "plan_course_task", "author_course_task", "review_course_task", "next_exercise"} {
|
|
if _, err := a.Repo.SaveProfile(ctx, user, aiTestProfile(action)); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
a.Repo.SaveBudget(ctx, user, 10000000, 0)
|
|
var authors atomic.Int32
|
|
var candidate any
|
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
var body struct {
|
|
Messages []struct {
|
|
Content string `json:"content"`
|
|
} `json:"messages"`
|
|
}
|
|
json.NewDecoder(r.Body).Decode(&body)
|
|
var q struct {
|
|
Task string `json:"task"`
|
|
Input json.RawMessage `json:"input"`
|
|
}
|
|
json.Unmarshal([]byte(body.Messages[1].Content), &q)
|
|
var result any
|
|
switch q.Task {
|
|
case "analyze_course_progress":
|
|
mods := []domain.ModuleAnalysis{}
|
|
for _, m := range store.EnglishCoursePack().Modules {
|
|
mods = append(mods, domain.ModuleAnalysis{ModuleID: m.ID, Strengths: []string{}, Weaknesses: []string{}, Recommendation: "Bereich ausprobieren", Reason: "Noch keine Nachweise", EvidenceIDs: []string{}})
|
|
}
|
|
result = domain.CourseAnalysis{SchemaVersion: 2, TargetLanguage: "en-US", CourseVersion: domain.CourseVersion, Summary: "Noch keine Nachweise.", RecommendedModuleID: "a1_introductions_v1", Modules: mods}
|
|
case "plan_course_task":
|
|
result = domain.CourseBrief{SchemaVersion: 2, ModuleID: "a1_family_v1", TargetLanguage: "en-US", CourseVersion: domain.CourseVersion, Skill: "grammar", Kind: "choice", Objective: "have und has unterscheiden", Difficulty: "A1", Reason: "Einstieg"}
|
|
case "author_course_task":
|
|
authors.Add(1)
|
|
task := store.EnglishCoursePack().Reserves[7]
|
|
task.ID = ""
|
|
task.Source = ""
|
|
candidate = map[string]any{"schema_version": 2, "tasks": []domain.AdaptiveTask{task}}
|
|
result = candidate
|
|
case "review_course_task":
|
|
var input struct {
|
|
Candidate struct {
|
|
Tasks []domain.AdaptiveTask `json:"tasks"`
|
|
} `json:"candidate"`
|
|
}
|
|
json.Unmarshal(q.Input, &input)
|
|
if len(input.Candidate.Tasks) != 1 {
|
|
t.Error("reviewer missing author output")
|
|
}
|
|
if input.Candidate.Tasks[0].AnswerKey != nil {
|
|
t.Error("reviewer was shown the authored solution")
|
|
}
|
|
result = AIReview{SchemaVersion: 2, Approved: true, ChoiceChecks: []AIChoiceCheck{aiChoiceCheck(store.EnglishCoursePack().Reserves[7], 0, store.EnglishCoursePack().Reserves[7].AnswerKey.CorrectOptionID)}, ReviewNotes: "Unabhängig gelöst"}
|
|
case "next_exercise":
|
|
result = map[string]any{"exercise_type": "repeat_sentence", "target_language": "en-US", "cefr_level": "A1", "reference_text": "My sister is here.", "display_prompt": "Sprich nach.", "tts_text": "My sister is here.", "focus_points": []string{"sister"}, "success_criteria": []string{"verständlich sprechen"}, "reason_for_choice": "Familienwort"}
|
|
default:
|
|
t.Errorf("unknown operation %s", q.Task)
|
|
}
|
|
json.NewEncoder(w).Encode(aiCompletion(result, true))
|
|
}))
|
|
defer server.Close()
|
|
a.URLs["mistral"] = server.URL
|
|
engine := NewCourseEngine(store.NewAdaptiveStore(db), profiles, a, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
|
job, err := engine.Repo.Enqueue(ctx, user, "en-US", "next", "a1_family_v1", "grammar", "", "direct-request")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
claimed, err := engine.Repo.Claim(ctx)
|
|
if err != nil || claimed.ID != job.ID {
|
|
t.Fatal(err)
|
|
}
|
|
generated, err := engine.Process(ctx, claimed)
|
|
if err != nil || generated.Source != "ai" || authors.Load() != 1 {
|
|
t.Fatal(generated, err)
|
|
}
|
|
stored, err := engine.Repo.Task(ctx, user, "en-US", generated.ID)
|
|
if err != nil || stored.Source != "ai" || stored.AnswerKey == nil {
|
|
t.Fatal("course provenance/storage", stored, err)
|
|
}
|
|
if err = engine.Repo.Finish(ctx, claimed, generated, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
exercise, err := a.NextExercise(ctx, domain.GeneratedExerciseRequest{UserID: user, TargetLanguage: "en-US", CEFRLevel: "A1"})
|
|
if err != nil || exercise.Source != "ai" {
|
|
t.Fatal(exercise, err)
|
|
}
|
|
if _, saved, err := profiles.SaveGeneratedExercise(ctx, user, exercise); err != nil || saved.Source != "ai" {
|
|
t.Fatal("pronunciation persistence", saved, err)
|
|
}
|
|
}
|
|
|
|
// A response timeout is intentionally not retried blindly: billing may have
|
|
// occurred, and the remaining budget reservation must remain visible.
|
|
func TestAICallDeadlineWithoutNetwork(t *testing.T) {
|
|
a, _, user := setupAI(t)
|
|
ctx := context.Background()
|
|
a.Repo.SaveProfile(ctx, user, aiTestProfile("evaluate_course_attempt"))
|
|
a.Repo.SaveBudget(ctx, user, 1000000, 0)
|
|
a.Client = &http.Client{Transport: speechRoundTripFunc(func(r *http.Request) (*http.Response, error) { return nil, context.DeadlineExceeded })}
|
|
work, _, err := a.Begin(ctx, user, "deadline", true)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var out domain.CourseEvaluation
|
|
start := time.Now()
|
|
if err = a.Call(work, "evaluate_course_attempt", map[string]string{"answer": "Hello"}, &out); err == nil {
|
|
t.Fatal("timeout ignored")
|
|
}
|
|
if time.Since(start) > time.Second {
|
|
t.Fatal("unexpected wait")
|
|
}
|
|
report, _ := a.Repo.Usage(ctx, domain.AIUsageFilter{})
|
|
if report.Total != 1 || report.Unknown != 1 || report.Reserved == 0 {
|
|
t.Fatal(report)
|
|
}
|
|
}
|
|
|
|
func TestAIPostgresDailyLimitAndUnknownResolution(t *testing.T) {
|
|
a, profiles, user := setupAI(t)
|
|
ctx := context.Background()
|
|
repo := a.Repo
|
|
p := aiTestProfile("evaluate_course_attempt")
|
|
saved, err := repo.SaveProfile(ctx, user, p)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = repo.SaveProfile(ctx, user, p); !errors.Is(err, store.ErrAIConflict) {
|
|
t.Fatal("stale settings overwrote profile", err)
|
|
}
|
|
if err = profiles.Init(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
all, err := repo.Profiles(ctx)
|
|
if err != nil || all[p.Action].Version != saved.Version || !all[p.Action].Enabled {
|
|
t.Fatal("migration reset settings", err)
|
|
}
|
|
if err = repo.SaveBudget(ctx, user, 1000000, 100); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
reserve := func(id string) (domain.AICall, error) {
|
|
_, err := repo.Operation(ctx, domain.AIOperation{ID: id, UserID: user})
|
|
if err != nil {
|
|
return domain.AICall{}, err
|
|
}
|
|
return repo.Reserve(ctx, domain.AICall{OperationID: id, UserID: user, Action: p.Action, Profile: saved, Reserved: 60})
|
|
}
|
|
c, err := reserve("daily-one")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = reserve("daily-two"); !errors.Is(err, store.ErrAIBudget) {
|
|
t.Fatal("daily limit ignored", err)
|
|
}
|
|
c.Status = "succeeded"
|
|
c.Output = `{"score":100,"verdict":"correct","feedback":"Gut","issues":[]}`
|
|
if err = repo.Finish(ctx, c); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Successful output with missing usage remains cached and can be reconciled.
|
|
if err = repo.Resolve(ctx, user, c.ID, 10, "Abgleich mit Anbieterrechnung"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = repo.Resolve(ctx, user, c.ID, 0, "Erneuter Abgleich"); !errors.Is(err, store.ErrAIConflict) {
|
|
t.Fatal("reconciliation overwritten", err)
|
|
}
|
|
cached, err := repo.Cached(ctx, c.OperationID, c.Action, c.InputHash)
|
|
if err != nil || cached == nil || cached.Cost == nil || *cached.Cost != 10 {
|
|
t.Fatal("reconciliation lost cached output", err)
|
|
}
|
|
if _, err = reserve("daily-two"); err != nil {
|
|
t.Fatal("released budget unavailable", err)
|
|
}
|
|
}
|
|
|
|
func TestAIPostgresExplicitRetryRefreshesFrozenProfile(t *testing.T) {
|
|
db, profiles := adaptiveTestDB(t)
|
|
ctx := context.Background()
|
|
account, err := profiles.CreateUser(ctx, "retry-ai@example.test", "hash", "de-DE")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a := NewAIService(store.NewAIStore(db), config.Config{MistralAPIKey: "fixture-secret"})
|
|
engine := NewCourseEngine(store.NewAdaptiveStore(db), profiles, a, slog.New(slog.NewTextHandler(io.Discard, nil)))
|
|
task, err := engine.Repo.Reserve(ctx, account.ID, "en-US", "a1_family_v1", "writing")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
attempt, job, err := engine.Submit(ctx, account.ID, "en-US", task.ID, "retry-saved-answer", "He has got a sister.", nil)
|
|
if err != nil || job == nil {
|
|
t.Fatal(err)
|
|
}
|
|
claimed, err := engine.Repo.Claim(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = engine.Process(ctx, claimed); !errors.Is(err, store.ErrAIDisabled) {
|
|
t.Fatal("disabled profile not detected", err)
|
|
}
|
|
engine.Repo.FailAttempt(ctx, account.ID, attempt.ID)
|
|
engine.Repo.Finish(ctx, claimed, nil, "disabled")
|
|
p := aiTestProfile("evaluate_course_attempt")
|
|
if _, err = a.Repo.SaveProfile(ctx, account.ID, p); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a.Repo.SaveBudget(ctx, account.ID, 1000000, 0)
|
|
provider := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
json.NewEncoder(w).Encode(aiCompletion(domain.CourseEvaluation{Score: 100, Verdict: "correct", Feedback: "Richtig", Issues: []string{}}, true))
|
|
}))
|
|
defer provider.Close()
|
|
a.URLs["mistral"] = provider.URL
|
|
retried, err := engine.Repo.RetryEvaluation(ctx, account.ID, "en-US", attempt.ID)
|
|
if err != nil || retried.AIRevision != 1 {
|
|
t.Fatal("new retry revision missing", retried, err)
|
|
}
|
|
claimed, err = engine.Repo.Claim(ctx)
|
|
if err != nil || claimed.ID != job.ID {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err = engine.Process(ctx, claimed); err != nil {
|
|
t.Fatal("retry retained unusable old settings", err)
|
|
}
|
|
preserved, err := engine.Repo.Attempt(ctx, account.ID, "en-US", attempt.ID)
|
|
if err != nil || preserved.Answer != "He has got a sister." || preserved.Evaluation == nil || preserved.Evaluation.Score != 100 {
|
|
t.Fatal("answer lost during retry", preserved, err)
|
|
}
|
|
}
|
|
|
|
func TestAIPostgresOperationDoesNotMutateSharedPrompts(t *testing.T) {
|
|
a, _, user := setupAI(t)
|
|
ctx := context.Background()
|
|
action := "evaluate_course_attempt"
|
|
old := a.Prompts[action]
|
|
if _, _, err := a.Begin(ctx, user, "frozen-prompts", true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
a.Prompts[action] = "new prompt version"
|
|
a.PromptVersion = "new-version"
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 8; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
_, frozen, err := a.Begin(ctx, user, "frozen-prompts", true)
|
|
if err != nil || frozen.Prompts[action] != old {
|
|
t.Error("old prompt snapshot lost", err)
|
|
}
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
if a.Prompts[action] != "new prompt version" {
|
|
t.Fatal("reading old operation overwrote shared application prompts")
|
|
}
|
|
}
|