Files
Speech/apps/api/internal/services/speech.go
T
2026-06-07 22:20:57 +00:00

291 lines
8.8 KiB
Go

package services
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"math"
"net/http"
"net/url"
"strings"
"time"
"speech-learning-api/internal/config"
"speech-learning-api/internal/domain"
)
type SpeechService interface {
AssessPronunciation(ctx context.Context, input PronunciationInput) (domain.PronunciationResult, error)
Synthesize(ctx context.Context, locale, voice, text string) ([]byte, string, error)
}
type PronunciationInput struct {
Audio []byte
ContentType string
TargetText string
Locale string
}
type AzureSpeechService struct {
cfg config.Config
client *http.Client
logger *slog.Logger
}
func NewAzureSpeechService(cfg config.Config, logger *slog.Logger) *AzureSpeechService {
return &AzureSpeechService{
cfg: cfg,
client: &http.Client{
Timeout: 45 * time.Second,
},
logger: logger,
}
}
func (s *AzureSpeechService) AssessPronunciation(ctx context.Context, input PronunciationInput) (domain.PronunciationResult, error) {
if input.TargetText == "" {
return domain.PronunciationResult{}, errors.New("target text is required")
}
if s.cfg.AzureMock || s.cfg.AzureSpeechKey == "" || s.cfg.AzureRegion == "" {
return mockPronunciation(input.TargetText), nil
}
endpoint := fmt.Sprintf("https://%s.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=%s",
s.cfg.AzureRegion, url.QueryEscape(valueOr(input.Locale, "en-US")))
assessmentConfig := map[string]any{
"ReferenceText": input.TargetText,
"GradingSystem": "HundredMark",
"Granularity": "Phoneme",
"Dimension": "Comprehensive",
"EnableMiscue": true,
"PhonemeAlphabet": "IPA",
}
assessmentJSON, _ := json.Marshal(assessmentConfig)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(input.Audio))
if err != nil {
return domain.PronunciationResult{}, err
}
req.Header.Set("Ocp-Apim-Subscription-Key", s.cfg.AzureSpeechKey)
req.Header.Set("Accept", "application/json;text/xml")
req.Header.Set("Content-Type", contentTypeForAzure(input.ContentType))
req.Header.Set("Pronunciation-Assessment", base64.StdEncoding.EncodeToString(assessmentJSON))
resp, err := s.client.Do(req)
if err != nil {
return domain.PronunciationResult{}, err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return domain.PronunciationResult{}, err
}
if resp.StatusCode >= 300 {
s.logger.Warn("azure pronunciation failed", "status", resp.StatusCode, "body", string(body))
return domain.PronunciationResult{}, fmt.Errorf("azure pronunciation status %d", resp.StatusCode)
}
return parseAzurePronunciation(body)
}
func (s *AzureSpeechService) Synthesize(ctx context.Context, locale, voice, text string) ([]byte, string, error) {
if strings.TrimSpace(text) == "" {
return nil, "", errors.New("tts text is required")
}
if s.cfg.AzureMock || s.cfg.AzureSpeechKey == "" || s.cfg.AzureRegion == "" {
return nil, "", nil
}
locale = valueOr(locale, "en-US")
voice = valueOr(voice, "en-US-JennyNeural")
ssml := fmt.Sprintf(`<speak version="1.0" xml:lang="%s"><voice name="%s">%s</voice></speak>`, locale, voice, xmlEscape(text))
endpoint := fmt.Sprintf("https://%s.tts.speech.microsoft.com/cognitiveservices/v1", s.cfg.AzureRegion)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(ssml))
if err != nil {
return nil, "", err
}
req.Header.Set("Ocp-Apim-Subscription-Key", s.cfg.AzureSpeechKey)
req.Header.Set("Content-Type", "application/ssml+xml")
req.Header.Set("X-Microsoft-OutputFormat", "audio-24khz-48kbitrate-mono-mp3")
req.Header.Set("User-Agent", "speech-learning-app")
resp, err := s.client.Do(req)
if err != nil {
return nil, "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, "", err
}
if resp.StatusCode >= 300 {
return nil, "", fmt.Errorf("azure tts status %d: %s", resp.StatusCode, string(body))
}
return body, "audio/mpeg", nil
}
type azurePronunciationResponse struct {
RecognitionStatus string `json:"RecognitionStatus"`
DisplayText string `json:"DisplayText"`
NBest []struct {
Display string `json:"Display"`
PronScore float64 `json:"PronScore"`
AccuracyScore float64 `json:"AccuracyScore"`
FluencyScore float64 `json:"FluencyScore"`
CompletenessScore float64 `json:"CompletenessScore"`
ProsodyScore float64 `json:"ProsodyScore"`
PronunciationAssessment struct {
PronScore float64 `json:"PronScore"`
AccuracyScore float64 `json:"AccuracyScore"`
FluencyScore float64 `json:"FluencyScore"`
CompletenessScore float64 `json:"CompletenessScore"`
ProsodyScore float64 `json:"ProsodyScore"`
} `json:"PronunciationAssessment"`
Words []struct {
Word string `json:"Word"`
ErrorType string `json:"ErrorType"`
AccuracyScore float64 `json:"AccuracyScore"`
PronunciationAssessment struct {
AccuracyScore float64 `json:"AccuracyScore"`
ErrorType string `json:"ErrorType"`
} `json:"PronunciationAssessment"`
Phonemes []struct {
Phoneme string `json:"Phoneme"`
} `json:"Phonemes"`
} `json:"Words"`
} `json:"NBest"`
}
func parseAzurePronunciation(body []byte) (domain.PronunciationResult, error) {
var raw azurePronunciationResponse
if err := json.Unmarshal(body, &raw); err != nil {
return domain.PronunciationResult{}, err
}
if len(raw.NBest) == 0 {
return domain.PronunciationResult{}, fmt.Errorf("azure response has no NBest result")
}
best := raw.NBest[0]
result := domain.PronunciationResult{
RecognizedText: firstNonEmpty(best.Display, raw.DisplayText),
OverallScore: firstScore(best.PronScore, best.PronunciationAssessment.PronScore),
AccuracyScore: firstScore(best.AccuracyScore, best.PronunciationAssessment.AccuracyScore),
FluencyScore: firstScore(best.FluencyScore, best.PronunciationAssessment.FluencyScore),
CompletenessScore: firstScore(best.CompletenessScore, best.PronunciationAssessment.CompletenessScore),
ProsodyScore: firstScore(best.ProsodyScore, best.PronunciationAssessment.ProsodyScore),
Provider: "azure",
}
for _, word := range best.Words {
phonemes := make([]string, 0, len(word.Phonemes))
for _, phoneme := range word.Phonemes {
if phoneme.Phoneme != "" {
phonemes = append(phonemes, phoneme.Phoneme)
}
}
score := firstScore(word.AccuracyScore, word.PronunciationAssessment.AccuracyScore)
result.WordScores = append(result.WordScores, domain.WordScore{
Word: word.Word,
Accuracy: score,
ErrorType: firstNonEmpty(word.PronunciationAssessment.ErrorType, word.ErrorType),
Phonemes: phonemes,
Highlight: highlightFor(score),
})
}
return result, nil
}
func mockPronunciation(target string) domain.PronunciationResult {
words := strings.Fields(target)
wordScores := make([]domain.WordScore, 0, len(words))
total := 0.0
for i, word := range words {
clean := strings.Trim(word, ".,!?;:")
score := 92.0 - float64((len(clean)+i*7)%32)
if strings.Contains(strings.ToLower(clean), "th") {
score -= 10
}
score = math.Max(48, math.Min(97, score))
total += score
wordScores = append(wordScores, domain.WordScore{
Word: clean,
Accuracy: score,
Highlight: highlightFor(score),
})
}
avg := 82.0
if len(wordScores) > 0 {
avg = total / float64(len(wordScores))
}
return domain.PronunciationResult{
RecognizedText: target,
OverallScore: avg,
AccuracyScore: avg,
FluencyScore: math.Min(98, avg+4),
CompletenessScore: math.Min(100, avg+6),
ProsodyScore: math.Max(50, avg-3),
WordScores: wordScores,
Provider: "mock",
}
}
func highlightFor(score float64) string {
switch {
case score >= 85:
return "good"
case score >= 70:
return "ok"
default:
return "weak"
}
}
func contentTypeForAzure(contentType string) string {
if strings.Contains(contentType, "wav") {
return "audio/wav; codecs=audio/pcm; samplerate=16000"
}
if strings.TrimSpace(contentType) != "" {
return contentType
}
return "audio/wav; codecs=audio/pcm; samplerate=16000"
}
func xmlEscape(value string) string {
value = strings.ReplaceAll(value, "&", "&amp;")
value = strings.ReplaceAll(value, "<", "&lt;")
value = strings.ReplaceAll(value, ">", "&gt;")
value = strings.ReplaceAll(value, `"`, "&quot;")
value = strings.ReplaceAll(value, "'", "&apos;")
return value
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
return value
}
}
return ""
}
func firstScore(values ...float64) float64 {
for _, value := range values {
if value != 0 {
return value
}
}
return 0
}
func valueOr(value, fallback string) string {
if strings.TrimSpace(value) == "" {
return fallback
}
return value
}