291 lines
8.8 KiB
Go
291 lines
8.8 KiB
Go
package services
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log/slog"
|
|
"math"
|
|
"net/http"
|
|
"net/url"
|
|
"strings"
|
|
"time"
|
|
|
|
"speech-learning-api/internal/config"
|
|
"speech-learning-api/internal/domain"
|
|
)
|
|
|
|
type SpeechService interface {
|
|
AssessPronunciation(ctx context.Context, input PronunciationInput) (domain.PronunciationResult, error)
|
|
Synthesize(ctx context.Context, locale, voice, text string) ([]byte, string, error)
|
|
}
|
|
|
|
type PronunciationInput struct {
|
|
Audio []byte
|
|
ContentType string
|
|
TargetText string
|
|
Locale string
|
|
}
|
|
|
|
type AzureSpeechService struct {
|
|
cfg config.Config
|
|
client *http.Client
|
|
logger *slog.Logger
|
|
}
|
|
|
|
func NewAzureSpeechService(cfg config.Config, logger *slog.Logger) *AzureSpeechService {
|
|
return &AzureSpeechService{
|
|
cfg: cfg,
|
|
client: &http.Client{
|
|
Timeout: 45 * time.Second,
|
|
},
|
|
logger: logger,
|
|
}
|
|
}
|
|
|
|
func (s *AzureSpeechService) AssessPronunciation(ctx context.Context, input PronunciationInput) (domain.PronunciationResult, error) {
|
|
if input.TargetText == "" {
|
|
return domain.PronunciationResult{}, errors.New("target text is required")
|
|
}
|
|
if s.cfg.AzureMock || s.cfg.AzureSpeechKey == "" || s.cfg.AzureRegion == "" {
|
|
return mockPronunciation(input.TargetText), nil
|
|
}
|
|
|
|
endpoint := fmt.Sprintf("https://%s.stt.speech.microsoft.com/speech/recognition/conversation/cognitiveservices/v1?language=%s",
|
|
s.cfg.AzureRegion, url.QueryEscape(valueOr(input.Locale, "en-US")))
|
|
|
|
assessmentConfig := map[string]any{
|
|
"ReferenceText": input.TargetText,
|
|
"GradingSystem": "HundredMark",
|
|
"Granularity": "Phoneme",
|
|
"Dimension": "Comprehensive",
|
|
"EnableMiscue": true,
|
|
"PhonemeAlphabet": "IPA",
|
|
}
|
|
assessmentJSON, _ := json.Marshal(assessmentConfig)
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(input.Audio))
|
|
if err != nil {
|
|
return domain.PronunciationResult{}, err
|
|
}
|
|
req.Header.Set("Ocp-Apim-Subscription-Key", s.cfg.AzureSpeechKey)
|
|
req.Header.Set("Accept", "application/json;text/xml")
|
|
req.Header.Set("Content-Type", contentTypeForAzure(input.ContentType))
|
|
req.Header.Set("Pronunciation-Assessment", base64.StdEncoding.EncodeToString(assessmentJSON))
|
|
|
|
resp, err := s.client.Do(req)
|
|
if err != nil {
|
|
return domain.PronunciationResult{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return domain.PronunciationResult{}, err
|
|
}
|
|
if resp.StatusCode >= 300 {
|
|
s.logger.Warn("azure pronunciation failed", "status", resp.StatusCode, "body", string(body))
|
|
return domain.PronunciationResult{}, fmt.Errorf("azure pronunciation status %d", resp.StatusCode)
|
|
}
|
|
return parseAzurePronunciation(body)
|
|
}
|
|
|
|
func (s *AzureSpeechService) Synthesize(ctx context.Context, locale, voice, text string) ([]byte, string, error) {
|
|
if strings.TrimSpace(text) == "" {
|
|
return nil, "", errors.New("tts text is required")
|
|
}
|
|
if s.cfg.AzureMock || s.cfg.AzureSpeechKey == "" || s.cfg.AzureRegion == "" {
|
|
return nil, "", nil
|
|
}
|
|
|
|
locale = valueOr(locale, "en-US")
|
|
voice = valueOr(voice, "en-US-JennyNeural")
|
|
ssml := fmt.Sprintf(`<speak version="1.0" xml:lang="%s"><voice name="%s">%s</voice></speak>`, locale, voice, xmlEscape(text))
|
|
endpoint := fmt.Sprintf("https://%s.tts.speech.microsoft.com/cognitiveservices/v1", s.cfg.AzureRegion)
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, strings.NewReader(ssml))
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
req.Header.Set("Ocp-Apim-Subscription-Key", s.cfg.AzureSpeechKey)
|
|
req.Header.Set("Content-Type", "application/ssml+xml")
|
|
req.Header.Set("X-Microsoft-OutputFormat", "audio-24khz-48kbitrate-mono-mp3")
|
|
req.Header.Set("User-Agent", "speech-learning-app")
|
|
|
|
resp, err := s.client.Do(req)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
body, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
if resp.StatusCode >= 300 {
|
|
return nil, "", fmt.Errorf("azure tts status %d: %s", resp.StatusCode, string(body))
|
|
}
|
|
return body, "audio/mpeg", nil
|
|
}
|
|
|
|
type azurePronunciationResponse struct {
|
|
RecognitionStatus string `json:"RecognitionStatus"`
|
|
DisplayText string `json:"DisplayText"`
|
|
NBest []struct {
|
|
Display string `json:"Display"`
|
|
PronScore float64 `json:"PronScore"`
|
|
AccuracyScore float64 `json:"AccuracyScore"`
|
|
FluencyScore float64 `json:"FluencyScore"`
|
|
CompletenessScore float64 `json:"CompletenessScore"`
|
|
ProsodyScore float64 `json:"ProsodyScore"`
|
|
PronunciationAssessment struct {
|
|
PronScore float64 `json:"PronScore"`
|
|
AccuracyScore float64 `json:"AccuracyScore"`
|
|
FluencyScore float64 `json:"FluencyScore"`
|
|
CompletenessScore float64 `json:"CompletenessScore"`
|
|
ProsodyScore float64 `json:"ProsodyScore"`
|
|
} `json:"PronunciationAssessment"`
|
|
Words []struct {
|
|
Word string `json:"Word"`
|
|
ErrorType string `json:"ErrorType"`
|
|
AccuracyScore float64 `json:"AccuracyScore"`
|
|
PronunciationAssessment struct {
|
|
AccuracyScore float64 `json:"AccuracyScore"`
|
|
ErrorType string `json:"ErrorType"`
|
|
} `json:"PronunciationAssessment"`
|
|
Phonemes []struct {
|
|
Phoneme string `json:"Phoneme"`
|
|
} `json:"Phonemes"`
|
|
} `json:"Words"`
|
|
} `json:"NBest"`
|
|
}
|
|
|
|
func parseAzurePronunciation(body []byte) (domain.PronunciationResult, error) {
|
|
var raw azurePronunciationResponse
|
|
if err := json.Unmarshal(body, &raw); err != nil {
|
|
return domain.PronunciationResult{}, err
|
|
}
|
|
if len(raw.NBest) == 0 {
|
|
return domain.PronunciationResult{}, fmt.Errorf("azure response has no NBest result")
|
|
}
|
|
best := raw.NBest[0]
|
|
result := domain.PronunciationResult{
|
|
RecognizedText: firstNonEmpty(best.Display, raw.DisplayText),
|
|
OverallScore: firstScore(best.PronScore, best.PronunciationAssessment.PronScore),
|
|
AccuracyScore: firstScore(best.AccuracyScore, best.PronunciationAssessment.AccuracyScore),
|
|
FluencyScore: firstScore(best.FluencyScore, best.PronunciationAssessment.FluencyScore),
|
|
CompletenessScore: firstScore(best.CompletenessScore, best.PronunciationAssessment.CompletenessScore),
|
|
ProsodyScore: firstScore(best.ProsodyScore, best.PronunciationAssessment.ProsodyScore),
|
|
Provider: "azure",
|
|
}
|
|
for _, word := range best.Words {
|
|
phonemes := make([]string, 0, len(word.Phonemes))
|
|
for _, phoneme := range word.Phonemes {
|
|
if phoneme.Phoneme != "" {
|
|
phonemes = append(phonemes, phoneme.Phoneme)
|
|
}
|
|
}
|
|
score := firstScore(word.AccuracyScore, word.PronunciationAssessment.AccuracyScore)
|
|
result.WordScores = append(result.WordScores, domain.WordScore{
|
|
Word: word.Word,
|
|
Accuracy: score,
|
|
ErrorType: firstNonEmpty(word.PronunciationAssessment.ErrorType, word.ErrorType),
|
|
Phonemes: phonemes,
|
|
Highlight: highlightFor(score),
|
|
})
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func mockPronunciation(target string) domain.PronunciationResult {
|
|
words := strings.Fields(target)
|
|
wordScores := make([]domain.WordScore, 0, len(words))
|
|
total := 0.0
|
|
for i, word := range words {
|
|
clean := strings.Trim(word, ".,!?;:")
|
|
score := 92.0 - float64((len(clean)+i*7)%32)
|
|
if strings.Contains(strings.ToLower(clean), "th") {
|
|
score -= 10
|
|
}
|
|
score = math.Max(48, math.Min(97, score))
|
|
total += score
|
|
wordScores = append(wordScores, domain.WordScore{
|
|
Word: clean,
|
|
Accuracy: score,
|
|
Highlight: highlightFor(score),
|
|
})
|
|
}
|
|
avg := 82.0
|
|
if len(wordScores) > 0 {
|
|
avg = total / float64(len(wordScores))
|
|
}
|
|
return domain.PronunciationResult{
|
|
RecognizedText: target,
|
|
OverallScore: avg,
|
|
AccuracyScore: avg,
|
|
FluencyScore: math.Min(98, avg+4),
|
|
CompletenessScore: math.Min(100, avg+6),
|
|
ProsodyScore: math.Max(50, avg-3),
|
|
WordScores: wordScores,
|
|
Provider: "mock",
|
|
}
|
|
}
|
|
|
|
func highlightFor(score float64) string {
|
|
switch {
|
|
case score >= 85:
|
|
return "good"
|
|
case score >= 70:
|
|
return "ok"
|
|
default:
|
|
return "weak"
|
|
}
|
|
}
|
|
|
|
func contentTypeForAzure(contentType string) string {
|
|
if strings.Contains(contentType, "wav") {
|
|
return "audio/wav; codecs=audio/pcm; samplerate=16000"
|
|
}
|
|
if strings.TrimSpace(contentType) != "" {
|
|
return contentType
|
|
}
|
|
return "audio/wav; codecs=audio/pcm; samplerate=16000"
|
|
}
|
|
|
|
func xmlEscape(value string) string {
|
|
value = strings.ReplaceAll(value, "&", "&")
|
|
value = strings.ReplaceAll(value, "<", "<")
|
|
value = strings.ReplaceAll(value, ">", ">")
|
|
value = strings.ReplaceAll(value, `"`, """)
|
|
value = strings.ReplaceAll(value, "'", "'")
|
|
return value
|
|
}
|
|
|
|
func firstNonEmpty(values ...string) string {
|
|
for _, value := range values {
|
|
if strings.TrimSpace(value) != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func firstScore(values ...float64) float64 {
|
|
for _, value := range values {
|
|
if value != 0 {
|
|
return value
|
|
}
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func valueOr(value, fallback string) string {
|
|
if strings.TrimSpace(value) == "" {
|
|
return fallback
|
|
}
|
|
return value
|
|
}
|