From 87684ba0fd296baf794b372a993c6b9176b7d1f9 Mon Sep 17 00:00:00 2001 From: Eugenius Naumenko Date: Wed, 4 Nov 2026 18:11:41 +0000 Subject: [PATCH] =?UTF-8?q?Audio-Monitoring=20f=C3=BCr=20sp=C3=A4tere=20Er?= =?UTF-8?q?probung=20hinter=20deaktivierter=20Option=20erhalten?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mithörpfad mit separater Lautstärke und Schalter in der gemeinsamen Aufnahme-Komponente vorbereiten. In allen bestehenden Aufnahmebereichen bleiben Bedienung und Mithörpfad standardmäßig deaktiviert; eine spätere Erprobung erfordert enableMonitoring. Validiert: Web-Build, synthetischer Audio-Monitoring-Browsertest inklusive ausgeblendetem Standard, Bank- und Aussprache-Browser-Smokes gegen lokale PostgreSQL-Fixtures, Bank-Audit und git diff --check. Keine Live-Audioqualität oder VPS-Bereitstellung geprüft. --- README.md | 1 + apps/web/src/app/styles.css | 52 ++++++++ apps/web/src/components/AudioRecorder.tsx | 55 ++++++++- apps/web/tests/README.md | 16 +++ apps/web/tests/audio-monitoring.e2e.cjs | 140 ++++++++++++++++++++++ 5 files changed, 261 insertions(+), 3 deletions(-) create mode 100644 apps/web/tests/audio-monitoring.e2e.cjs diff --git a/README.md b/README.md index 19039f5..26c79fa 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Betriebsanleitung: [docs/AI_OPERATIONS.md](docs/AI_OPERATIONS.md). Die echte `.e - Registrierung, Login und Lernerprofil mit Zielsprache, Oberflächensprache und Tagesziel - Kurze englische `repeat_sentence`-Übungen aus direkten Modellaufrufen; Seed-Übungen als Fallback - Audioaufnahme im Browser und Aussprachebewertung über Azure Speech +- Audio-Monitoring ist für eine spätere Erprobung im Code erhalten, derzeit aber in allen Aufnahmebereichen ausgeblendet und deaktiviert. Bei einem manuellen Test mit USB-Headset unter Debian/Firefox wurden etwa 900 ms Verzögerung berichtet; die Ursache ist nicht geklärt. Eine spätere Freigabe erfolgt ausdrücklich über die Komponenten-Option `enableMonitoring`; Schalter und Mithörlautstärke (0–100 %) bleiben vorbereitet. - Gesamt-, Wort- und bei echten Azure-Antworten auch Phonemwerte - Feedback zum Versuch sowie Fortschrittsansicht mit schwachen Wörtern und Lauten - Mock-Modus für lokale Entwicklung ohne Azure-Zugang diff --git a/apps/web/src/app/styles.css b/apps/web/src/app/styles.css index 56d9330..1e0694c 100644 --- a/apps/web/src/app/styles.css +++ b/apps/web/src/app/styles.css @@ -317,6 +317,58 @@ input:focus { gap: 8px; } +.audio-monitoring { + display: grid; + gap: 10px; + max-width: 100%; + padding: 12px; + border: 1px solid #cfd8d3; + border-radius: 8px; +} + +.monitoring-toggle { + display: flex; + align-items: center; + gap: 10px; + position: relative; + cursor: pointer; +} + +.monitoring-toggle input { + position: absolute; + width: 1px; + height: 1px; + opacity: 0; + padding: 0; +} + +.monitoring-switch { + width: 42px; + height: 24px; + flex-shrink: 0; + border-radius: 12px; + background: #6b776f; +} + +.monitoring-switch::before { + content: ''; + display: block; + width: 18px; + height: 18px; + margin: 3px; + border-radius: 50%; + background: #fff; + transition: transform 0.15s; +} + +.monitoring-toggle input:checked + .monitoring-switch { background: #0e5f47; } +.monitoring-toggle input:checked + .monitoring-switch::before { transform: translateX(18px); } +.monitoring-toggle input:focus-visible + .monitoring-switch { outline: 3px solid #0e5f47; outline-offset: 3px; } +.monitoring-toggle:has(input:disabled) { cursor: not-allowed; opacity: 0.55; } +.monitoring-volume input { padding: 0; accent-color: #0e5f47; } +.monitoring-volume input:disabled { opacity: 0.55; } +.audio-monitoring p { margin: 0; font-size: 0.85rem; } + .recording-preview audio { max-width: 100%; } diff --git a/apps/web/src/components/AudioRecorder.tsx b/apps/web/src/components/AudioRecorder.tsx index 05cc21c..c6b05c0 100644 --- a/apps/web/src/components/AudioRecorder.tsx +++ b/apps/web/src/components/AudioRecorder.tsx @@ -1,5 +1,5 @@ import { Mic, Square } from 'lucide-react'; -import { useEffect, useRef, useState } from 'react'; +import { useEffect, useId, useRef, useState } from 'react'; type Props = { onRecorded: (blob: Blob) => void; @@ -9,12 +9,16 @@ type Props = { buttonLabel?: string; secondary?: boolean; maxSeconds?: number; + enableMonitoring?: boolean; }; -export function AudioRecorder({ onRecorded, onRecordingStart, onRecordingChange, disabled, buttonLabel = 'Aufnehmen', secondary, maxSeconds }: Props) { +export function AudioRecorder({ onRecorded, onRecordingStart, onRecordingChange, disabled, buttonLabel = 'Aufnehmen', secondary, maxSeconds, enableMonitoring = false }: Props) { const [recording, setRecording] = useState(false); const [starting, setStarting] = useState(false); const [error, setError] = useState(''); + const [monitoring, setMonitoring] = useState(false); + const [monitorVolume, setMonitorVolume] = useState(50); + const monitoringHintID = useId(); const startingRef = useRef(false); const recordingRef = useRef(false); const mountedRef = useRef(true); @@ -22,6 +26,8 @@ export function AudioRecorder({ onRecorded, onRecordingStart, onRecordingChange, const contextRef = useRef(null); const processorRef = useRef(null); const sourceRef = useRef(null); + const monitorRef = useRef(null); + const monitorLevel = useRef(0); const stopTimer = useRef(undefined); const chunks = useRef([]); const sampleRate = useRef(44100); @@ -31,11 +37,20 @@ export function AudioRecorder({ onRecorded, onRecordingStart, onRecordingChange, return () => { mountedRef.current = false; window.clearTimeout(stopTimer.current); + monitorRef.current?.disconnect(); streamRef.current?.getTracks().forEach((track) => track.stop()); void contextRef.current?.close(); }; }, []); + useEffect(() => { + monitorLevel.current = enableMonitoring && monitoring ? monitorVolume / 100 : 0; + const context = contextRef.current; + if (context && monitorRef.current) { + monitorRef.current.gain.setTargetAtTime(monitorLevel.current, context.currentTime, 0.01); + } + }, [enableMonitoring, monitoring, monitorVolume]); + async function start() { if (startingRef.current || recordingRef.current || disabled) return; startingRef.current = true; @@ -51,9 +66,23 @@ export function AudioRecorder({ onRecorded, onRecordingStart, onRecordingChange, onRecordingChange?.(false); return; } - audioContext = new AudioContext(); + audioContext = new AudioContext({ latencyHint: 'interactive' }); + await audioContext.resume(); + if (!mountedRef.current) { + stream.getTracks().forEach((track) => track.stop()); + void audioContext.close(); + onRecordingChange?.(false); + return; + } const source = audioContext.createMediaStreamSource(stream); const processor = audioContext.createScriptProcessor(4096, 1, 1); + // Monitor the microphone on a separate path: its volume must never change the recorded samples. + const monitor = enableMonitoring ? audioContext.createGain() : null; + if (monitor) { + monitor.gain.value = monitorLevel.current; + source.connect(monitor); + monitor.connect(audioContext.destination); + } chunks.current = []; sampleRate.current = audioContext.sampleRate; processor.onaudioprocess = (event) => { @@ -66,6 +95,7 @@ export function AudioRecorder({ onRecorded, onRecordingStart, onRecordingChange, contextRef.current = audioContext; processorRef.current = processor; sourceRef.current = source; + monitorRef.current = monitor; recordingRef.current = true; setRecording(true); onRecordingStart?.(); @@ -87,10 +117,12 @@ export function AudioRecorder({ onRecorded, onRecordingStart, onRecordingChange, window.clearTimeout(stopTimer.current); processorRef.current?.disconnect(); sourceRef.current?.disconnect(); + monitorRef.current?.disconnect(); streamRef.current?.getTracks().forEach((track) => track.stop()); void contextRef.current?.close(); processorRef.current = null; sourceRef.current = null; + monitorRef.current = null; streamRef.current = null; contextRef.current = null; setRecording(false); @@ -116,6 +148,23 @@ export function AudioRecorder({ onRecorded, onRecordingStart, onRecordingChange, {starting ? 'Mikrofon wird gestartet...' : buttonLabel} )} + {enableMonitoring &&
+ + +

Während der Aufnahme mit Kopfhörern nutzen. Die eigene Stimme kann leicht verzögert sein.

+
} {error && {error}} ); diff --git a/apps/web/tests/README.md b/apps/web/tests/README.md index 78a7785..d93da0b 100644 --- a/apps/web/tests/README.md +++ b/apps/web/tests/README.md @@ -1,5 +1,21 @@ # Browserprüfungen für Aufgabenbestand und KI-Anbindung +`audio-monitoring.e2e.cjs` prüft die gemeinsame Aufnahme-Komponente mit einem +synthetischen Mikrofonsignal: standardmäßig keine Mithörbedienung und kein +Mithörpfad; nach expliziter Freigabe über `enableMonitoring` zunächst ausgeschaltetes Mithören, Schalter und +Lautstärke während der Aufnahme, tatsächliche Signalpegel, unveränderte +WAV-Aufnahme, Stoppen und Unmount, Mikrofonfehler mit erneutem Versuch sowie +automatisches Zeitlimit und schmale Ansicht. Benötigt Web-Abhängigkeiten und +Playwright mit Chromium, aber keine API oder Datenbank. Aus `apps/web`: + +```bash +PLAYWRIGHT_MODULE=/pfad/zu/playwright node tests/audio-monitoring.e2e.cjs +``` + +Auch diese Prüfung verwendet Port 15175 und muss separat laufen. Sie bestätigt +keine echte Kopfhörerqualität oder Gerätelatenz; diese müssen mit echter +Hardware geprüft werden. + `pronunciation-profile.e2e.cjs` prüft das gemeinsame Lautprofil über echte lokale API-/PostgreSQL-Aufnahmen mit ausdrücklich simulierten Speech-Messungen: leerer Einstieg, Nachweise speichern, Wort-/Phonemübersicht, geringe Evidenz, diff --git a/apps/web/tests/audio-monitoring.e2e.cjs b/apps/web/tests/audio-monitoring.e2e.cjs new file mode 100644 index 0000000..3cd7414 --- /dev/null +++ b/apps/web/tests/audio-monitoring.e2e.cjs @@ -0,0 +1,140 @@ +// Local component fixture with a synthetic microphone; no API, Azure or real microphone. +const assert = require('node:assert/strict'); +const path = require('node:path'); +const { pathToFileURL } = require('node:url'); +const { chromium } = require(process.env.PLAYWRIGHT_MODULE || 'playwright'); + +(async () => { + const web = path.resolve(__dirname, '..'); + const { createServer } = await import(pathToFileURL(path.join(web, 'node_modules/vite/dist/node/index.js'))); + const server = await createServer({ + root: web, configFile: false, + plugins: [{ name: 'recorder-fixture', configureServer(dev) { dev.middlewares.use('/__recorder-test', recorderFixture); } }], + optimizeDeps: { include: ['react', 'react-dom/client', 'react/jsx-dev-runtime', 'lucide-react'] }, + server: { host: '127.0.0.1', port: 15175, strictPort: true } + }); + function recorderFixture(_req, res) { + res.setHeader('Content-Type', 'text/html'); + res.end(`
`); + } + let browser; + try { + await server.listen(); + browser = await chromium.launch({ headless: true, args: ['--no-sandbox', '--autoplay-policy=no-user-gesture-required'] }); + const page = await browser.newPage(); + const errors = []; + page.on('pageerror', error => { errors.push(error.message); console.error(error.message); }); + await page.addInitScript(() => { + const RealContext = window.AudioContext; + window.contexts = []; + window.monitors = []; + window.streams = []; + window.AudioContext = class extends RealContext { + constructor(options) { super(options); window.contexts.push(this); } + createGain() { + const gain = super.createGain(); + const analyser = super.createAnalyser(); + gain.connect(analyser); + window.monitors.push({ gain, analyser }); + return gain; + } + }; + navigator.mediaDevices.getUserMedia = async () => { + if (window.rejectMic) throw new Error('Test: Mikrofon verweigert'); + const context = new RealContext(); + await context.resume(); + const oscillator = context.createOscillator(); + oscillator.frequency.value = 440; + const destination = context.createMediaStreamDestination(); + oscillator.connect(destination); + oscillator.start(); + window.streams.push(destination.stream); + destination.stream.getTracks()[0].addEventListener('ended', () => context.close()); + return destination.stream; + }; + window.monitorRMS = () => { + const data = new Float32Array(2048); + window.monitors.at(-1).analyser.getFloatTimeDomainData(data); + return Math.sqrt(data.reduce((sum, value) => sum + value * value, 0) / data.length); + }; + }); + await page.goto('http://127.0.0.1:15175/__recorder-test'); + const toggle = page.getByRole('switch', { name: 'Eigene Stimme mithören' }); + const volume = page.getByRole('slider', { name: 'Mithörlautstärke' }); + await page.getByRole('button', { name: 'Aufnehmen', exact: true }).waitFor(); + assert.equal(await toggle.count(), 0); + assert.equal(await volume.count(), 0); + await page.getByRole('button', { name: 'Aufnehmen', exact: true }).click(); + await page.getByRole('button', { name: 'Stoppen' }).waitFor(); + assert.equal(await page.evaluate(() => window.monitors.length), 0); + await page.getByRole('button', { name: 'Stoppen' }).click(); + await page.waitForFunction(() => window.contexts.every(c => c.state === 'closed')); + await page.evaluate(() => { window.blobs = []; window.streams = []; window.renderRecorder(undefined, true); }); + await toggle.waitFor(); + assert.equal(await toggle.isChecked(), false); + assert.equal(await volume.isDisabled(), true); + assert.equal(await page.evaluate(() => window.streams.length), 0); + await page.getByRole('button', { name: 'Aufnehmen', exact: true }).click(); + await page.getByRole('button', { name: 'Stoppen' }).waitFor(); + await page.waitForFunction(() => window.monitorRMS() === 0); + await toggle.check(); + await page.waitForFunction(() => window.monitorRMS() > 0.3); + await volume.fill('20'); + await page.waitForFunction(() => window.monitorRMS() > 0.1 && window.monitorRMS() < 0.2); + await toggle.uncheck(); + await page.waitForFunction(() => window.monitorRMS() < 0.001); + await page.getByRole('button', { name: 'Stoppen' }).click(); + await page.waitForFunction(() => window.contexts.every(c => c.state === 'closed')); + assert(await page.evaluate(() => window.streams.every(s => s.getTracks().every(t => t.readyState === 'ended')))); + // The WAV must retain microphone amplitude through muted and quiet monitoring segments. + const recording = await page.evaluate(async () => { + const buffer = await window.blobs[0].arrayBuffer(); + const view = new DataView(buffer); + const rms = []; + for (let start = 44; start + 3200 <= buffer.byteLength; start += 3200) { + let energy = 0; + for (let i = start; i < start + 3200; i += 2) energy += (view.getInt16(i, true) / 32768) ** 2; + rms.push(Math.sqrt(energy / 1600)); + } + return { rate: view.getUint32(24, true), channels: view.getUint16(22, true), rms }; + }); + assert.equal(recording.rate, 16000); + assert.equal(recording.channels, 1); + // Ignore the first 200 ms while the synthetic input stream starts delivering frames. + assert(recording.rms.length > 2); + assert(recording.rms.slice(2).every(rms => rms > 0.6 && rms < 0.8), JSON.stringify(recording.rms)); + await toggle.check(); + await page.getByRole('button', { name: 'Aufnehmen', exact: true }).click(); + await page.waitForFunction(() => window.monitors.length === 2 && window.monitorRMS() > 0.1); + await page.evaluate(() => window.unmountRecorder()); + await page.waitForFunction(() => window.contexts.every(c => c.state === 'closed')); + assert(await page.evaluate(() => window.streams.every(s => s.getTracks().every(t => t.readyState === 'ended')))); + await page.reload(); + await page.getByRole('button', { name: 'Aufnehmen', exact: true }).waitFor(); + await page.evaluate(() => window.rejectMic = true); + await page.getByRole('button', { name: 'Aufnehmen', exact: true }).click(); + await page.getByText('Test: Mikrofon verweigert', { exact: true }).waitFor(); + await page.evaluate(() => { window.rejectMic = false; window.renderRecorder(1); }); + await page.getByRole('button', { name: 'Aufnehmen', exact: true }).click(); + await page.waitForFunction(() => window.blobs.length === 1 && window.contexts.every(c => c.state === 'closed')); + assert.deepEqual(errors, []); + await page.setViewportSize({ width: 320, height: 800 }); + assert(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth)); + console.log('PASS: monitoring hidden and no monitor path by default; opt-in live switch/volume and actual signal levels, unchanged WAV, stop/unmount cleanup, microphone error/retry, time limit and mobile width. Synthetic audio only.'); + } finally { + if (browser) await browser.close(); + await server.close(); + } +})().catch(error => { console.error(error); process.exitCode = 1; });