diff --git a/AGENTS.md b/AGENTS.md index 2f2929e..7d6d933 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -180,6 +180,18 @@ Nachfolgende Benutzerkorrektur am selben Tag: Die zusätzliche CO₂-Anzeige wur - 60 Sekunden Monitor: Vier-Seiten-Navigation bereit, SEN66-Messungen und 55 erfolgreiche Influx-Antworten mit HTTP 204. Keine erkannten Panics oder Sensor-/Touch-Fehler. Zwei bekannte SD-Timeout-Fehlerzeilen, keine sonstigen Fehlerzeilen. Physische Wischgesten, sichtbares Displaybild und Speichern/Neuladen der Einstellungen sind am Gerät durch den Benutzer zu prüfen; keine entsprechende Hardwarebetätigung im Monitor beobachtet. - Geschützte Backups und Logs außerhalb des Repositorys: `flash-plan.json`, `backup-report.json`, `write-results.json`, `deployment-report.json`, Vor-/Nach-Flashdateien, Monitor, Build-/UI-Testlogs und Quellen-Hashes. Ausführung abgeschlossen; keine weitere Schreib-/Löschfreigabe ableiten. +## InfluxDB-Bündelung und Verbindungswiederverwendung (11.11.2026) + +- Benutzerauftrag: effizientere Übertragung durch größere Sammelanfragen. Messung und Display bleiben bei nominal 1 Hz; keine Mittelung der Influx-Messpunkte oder Änderung ihrer Felder. Bis zu zehn Punkte / 4096 Bytes pro Anfrage, Restpakete nach maximal zehn Sekunden. Im Normalbetrieb etwa sechs Schreibanfragen pro Minute statt 60; serverseitige Sichtbarkeit entsprechend verzögert. +- Asynchrone API serialisiert beim Einreihen in eine feste Warteschlange mit 32 Einträgen. Keine einzelnen Heap-Allokationen pro Feld. Worker-Stack 8192 Bytes. HTTP-Client/Verbindung werden wiederverwendet; Fehler oder geänderte Konfiguration bauen den Handle neu auf. Token und Payloads werden nicht protokolliert. +- Einzelne UTC-Zeitstempel mit `precision=ms` verhindern Zusammenfallen gebündelter Punkte derselben Serie. Messzeit wird monoton beim Einreihen erfasst. Ein GET `/health` richtet die private Messuhr am HTTP-Date desselben Servers aus (ungefähr sekundengenaue absolute UTC-Zuordnung, kein externer Zeitdienst/kein Stellen der Systemuhr). Ohne gültige Zeitbasis sichere Einzelpunkt-Anfragen statt ungestempelter Mehrpunktpakete. Explizite API-Zeitstempel bleiben erhalten. +- Transport-/Serverfehler behalten Paketinhalt und Zeitstempel für Wiederholungen bei; exponentielle Pause 2 s bis 300 s, Konfiguration nach fünf Fehlern erneut lesen. HTTP 400/413/422 verwirft das abgelehnte Paket mit Meldung. Ohne Zeitbasis werden fehlgeschlagene Einzelpunkte nicht automatisch mit neuer Serverzeit wiederholt. Volle Queue ersetzt die ältesten wartenden Punkte und zählt die Verluste. RAM-Inhalte gehen bei Reset verloren. Details/API-/Testhinweise stehen in `firmware/HARDWARE.md`. +- Zehn native Szenarien mit echtem Worker und FreeRTOS-/HTTP-Simulation unter Address-/UndefinedBehavior-Sanitizer bestanden: Formatierung/escaping/Sentinels, UTC/Kalender, 10er-Pakete, identische Wiederholungen, volle Queue, fehlende Zeitbasis, Einzelpunktfehler, Restpakete, Bytegrenze und abgelehnte Pakete. Client-Lebensdauer und eindeutige Messzeitstempel geprüft. Vollständiger ESP-IDF-5.5-Build im Environment `esp32-c6-devkitc-1` erfolgreich. +- Ziel `CC:BA:97:F2:5F:7C`, ESP32-C6FH8/v0.2/8MB erneut per `list`/`info`/`flash-id` bestätigt. Frisches vollständiges Backup mit unabhängigen Anfangs-/Endproben, Tabelle/MD5, OTA-Auswahl, Bootloader und Sektorrand geprüft. Genau ein App-Schreibvorgang bei `0x20000`, 1.846.464 Bytes, SHA256 `734654f29e16a20aa43086131115cc06d16b09909f55c2bbb14b4e4704a063a7`. Vollständiges bytegenaues/SHA256-Rücklesen erfolgreich, kein vollständiges Löschen. +- Vollständiger anschließender 8-MB-Vergleich: Bootloader, Partitionstabellensektor, OTA-Auswahl, PHY, übrige Flashbereiche und 832-Byte-Sektorrand unverändert. NVS änderte sich zur Laufzeit um 2067 Rohbytes; logische Inhalte diesmal nicht separat verglichen. Neue Influx-Logik liest die vorhandene Konfiguration und schreibt selbst keine NVS-Einstellungen. +- 60 Sekunden Monitor: fünf bestätigte Schreibanfragen mit 10/10/8/10/10 Punkten (48 Punkte gesamt), ein Verbindungsaufbau inklusive initialem Health-Request. Private Messuhr erfolgreich ausgerichtet; protokollierte Paketzeitstempel unterscheiden die Messzeitpunkte. Keine Schreibfehler, keine erkannten Panics, Sensor-/Touch-Fehler oder gezählten Queue-Verluste. Zwei bekannte SD-Timeout-Zeilen, keine anderen Fehlerzeilen. Kleinere Pakete bei Zeitlimit sind vorgesehen; keine Messwerte für fehlende Sensorpakete erfinden. +- Berichte/geschützte Logs außerhalb des Repositorys: `flash-plan.json`, `backup-report.json`, `write-results.json`, `deployment-report.json`, vollständige Flashdateien, Monitor, Build-/Testlogs und Quellen-Hashes. Diese Ausführung ist abgeschlossen; keine weitere automatische Schreib-/Löschfreigabe ableiten. + ## Bekannte Anwendungsprobleme (historischer Stand) Beim letzten Hardwaretest wurden festgestellt: diff --git a/firmware/HARDWARE.md b/firmware/HARDWARE.md index 0bd5a22..a136461 100644 --- a/firmware/HARDWARE.md +++ b/firmware/HARDWARE.md @@ -242,3 +242,58 @@ bereits aufgelöste LVGL-Komponente unter `firmware/managed_components`. Ein optionaler Verzeichnispfad hinter dem UI-Testbefehl speichert vier gerenderte PPM-Vorschauen mit simulierten Messwerten. Physische Darstellung und Fingerbedienung müssen zusätzlich am Gerät geprüft werden. + +## Effiziente InfluxDB-Übertragung + +Der Sensor misst weiter mit 1 Hz. Der Influx-Worker bündelt bis zu zehn einzelne +Messpunkte in einer HTTP-Anfrage; spätestens zehn Sekunden nach Beginn eines +Pakets wird auch ein kleineres Restpaket versendet. Ein Paket ist zusätzlich auf +4096 Bytes begrenzt. Keine Mittelung oder Änderung der Influx-Feldnamen/Einheiten; +Display und lokaler Verlauf aktualisieren sich unabhängig vom Netzwerk weiter. +Im normalen 1-Hz-Betrieb entstehen etwa sechs Schreibanfragen pro Minute statt 60, +mit bis zu etwa zehn Sekunden zusätzlicher Sichtbarkeitsverzögerung auf dem Server. + +Jeder Punkt erhält einen eigenen Zeitstempel mit `precision=ms`. Der Messzeitpunkt +wird beim Einreihen über den monotonen ESP-Timer erfasst. Ein einmaliger GET auf +`/health` liest den HTTP-`Date`-Header desselben Influx-Servers und richtet damit +nur die interne Messuhr aus; weder ein Internet-Zeitdienst noch eine Änderung der +Systemuhr ist nötig. HTTP Date besitzt Sekundenauflösung: Die absolute Zuordnung +ist entsprechend ungefähr sekundengenau; die Abstände der Sekundenmessungen +bleiben erhalten. Spätere Date-Header gleichen größere Uhrdrift vorsichtig aus. +Explizite Zeitstempel der API bleiben erhalten. Fehlt ein gültiges Date vollständig, +werden sichere Einzelpunkt-Anfragen mit Serverzeit verwendet, bis eine Zeitbasis +verfügbar ist. Mehrere ungestempelte Punkte derselben Serie werden nicht gebündelt. + +HTTP-Client und Verbindung werden über mehrere Anfragen wiederverwendet. Ein +Konfigurationswechsel oder Verbindungsfehler verwirft den Handle; der nächste +Versuch baut ihn neu auf. Es werden keine Token oder Messwert-Payloads protokolliert. +Der Worker meldet HTTP-Status, Punkt-/Bytezahl, erste/letzte UTC-Zeit im Paket und +gezählte Verbindungsaufbauten. Die Gerätestatusanzeige zählt nur bestätigte HTTP 204 +als erfolgreiche Übertragung; ihr Alter kann jetzt regulär etwa zehn Sekunden betragen. + +Der RAM-Puffer enthält 32 vorserialisierte Punkte ohne einzelne Heap-Allokationen +für jeden Feldnamen/Wert. Ein Paket bleibt bei Transportfehlern, Rate-Limits oder +Serverproblemen mit unverändertem Inhalt und Zeitstempeln für einen erneuten +Versuch erhalten (exponentielle Pause von 2 s bis maximal 300 s). Fünf Fehler lösen +ein Neuladen der vorhandenen Konfiguration aus. HTTP 400/413/422 wird als abgelehntes +Paket protokolliert und verworfen; bereits teilweise geschriebene Daten werden dabei +nicht gelöscht. Ohne Zeitbasis werden fehlgeschlagene Einzelpunkte nicht automatisch +wiederholt, da deren Serverzeit sonst neue Punkte erzeugen könnte. + +Bei längerem Ausfall ersetzt die volle Warteschlange die ältesten noch nicht +versendeten Punkte; die Verluste werden gezählt und protokolliert. Puffer und +laufendes Paket sind RAM-basiert und gehen bei Neustart verloren. Der Producer +wartet nicht auf HTTP. Beim Stoppen räumt der Worker seine Ressourcen selbst auf; +er wird nicht mitten in einer Anfrage gewaltsam beendet. + +Nativer Test mit dem echten Worker und simuliertem FreeRTOS/HTTP unter Address- und +UndefinedBehavior-Sanitizer, vom Repository-Hauptverzeichnis: + +```bash +python3 firmware/test/net_influx/run_tests.py +``` + +Prüft Serialisierung/escaping, fehlende Kanäle, UTC-/Kalenderkonvertierung, +Messzeitabstände, 10er-Pakete, Wiederholungen mit identischem Body, Warteschlangen- +Überlauf, fehlende Zeitbasis, Einzelpunktfehler, Zeitlimit bei Restpaketen, +Bytegrenzen, dauerhaft abgelehnte Pakete und Wiederverwendung/Freigabe des Clients. diff --git a/firmware/components/net_influx/CMakeLists.txt b/firmware/components/net_influx/CMakeLists.txt index 5d45cdf..4f96f52 100644 --- a/firmware/components/net_influx/CMakeLists.txt +++ b/firmware/components/net_influx/CMakeLists.txt @@ -1,6 +1,7 @@ idf_component_register( SRCS "src/net_influx.c" + "src/influx_batch.c" INCLUDE_DIRS "include" REQUIRES @@ -10,5 +11,6 @@ idf_component_register( storage esp_http_client sys_status + esp_timer ) diff --git a/firmware/components/net_influx/include/net_influx.h b/firmware/components/net_influx/include/net_influx.h index bd5b8cc..3bf5aa3 100644 --- a/firmware/components/net_influx/include/net_influx.h +++ b/firmware/components/net_influx/include/net_influx.h @@ -2,6 +2,8 @@ #pragma once #include #include +#include +#include "freertos/FreeRTOS.h" #include "esp_err.h" typedef enum { @@ -28,15 +30,15 @@ typedef struct { typedef struct { const char* measurement; // z.B. "air" const char* tags; // z.B. "device=xyz,location=lab" (optional, kann NULL sein) - uint64_t ts_ms; // 0 => server time + uint64_t ts_ms; // 0 => acquisition time aligned to the server HTTP Date (async API) const net_influx_field_t* fields; size_t num_fields; } net_influx_point_t; typedef struct { // Task/Queue - uint16_t queue_len; // z.B. 16 - uint32_t task_stack; // z.B. 6144 + uint16_t queue_len; // Standard 32 + uint32_t task_stack; // Standard 8192 UBaseType_t task_prio; // z.B. 5 // Init/Retry/Backoff @@ -68,7 +70,9 @@ esp_err_t net_influx_init_from_nvs(const char* ca_cert_pem_opt, int timeout_ms); esp_err_t net_influx_start(const net_influx_runtime_cfg_t* rt); void net_influx_stop(void); -// Asynchron: kopiert point+felder intern und queued sie. +// Asynchron: serialisiert den Punkt sofort in eine feste RAM-Warteschlange. +// 1-Hz-Werte werden bis zu 10 Punkte / 10 s gesammelt, mit eigenen Zeitstempeln. +// Ohne gültige Serverzeit werden sichere Einzelpunkt-Schreibvorgänge verwendet. // Rückgabe: // - ESP_OK: queued // - ESP_ERR_NO_MEM / ESP_ERR_INVALID_ARG / ESP_FAIL: sonst diff --git a/firmware/components/net_influx/src/influx_batch.c b/firmware/components/net_influx/src/influx_batch.c new file mode 100644 index 0000000..e230a3d --- /dev/null +++ b/firmware/components/net_influx/src/influx_batch.c @@ -0,0 +1,87 @@ +#include "influx_batch.h" +#include +#include +#include + +static bool leap(unsigned year) { return year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); } + +bool influx_parse_http_date(const char *date, uint64_t *epoch_ms) +{ + if (!date || !epoch_ms || strlen(date) != 29) return false; + char weekday[4], month[4], zone[4], extra; + unsigned day, year, hour, minute, second; + if (sscanf(date, "%3s, %u %3s %u %u:%u:%u %3s%c", weekday, &day, month, + &year, &hour, &minute, &second, zone, &extra) != 8) return false; + if (strcmp(zone, "GMT") || year < 2020 || year > 2261 || hour > 23 || minute > 59 || second > 59) return false; + const char *months[] = {"Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"}; + const unsigned days[] = {31,28,31,30,31,30,31,31,30,31,30,31}; + unsigned m = 0; + while (m < 12 && strcmp(month, months[m])) ++m; + if (m == 12 || day == 0 || day > days[m] + (m == 1 && leap(year))) return false; + uint64_t total = 0; + for (unsigned y = 1970; y < year; ++y) total += leap(y) ? 366 : 365; + for (unsigned i = 0; i < m; ++i) total += days[i] + (i == 1 && leap(year)); + total += day - 1; + *epoch_ms = ((total * 24 + hour) * 60 * 60 + minute * 60 + second) * 1000; + return true; +} + +static uint64_t clock_at(const influx_clock_t *clock, uint64_t mono_ms) +{ + return mono_ms >= clock->monotonic_ms ? clock->epoch_ms + mono_ms - clock->monotonic_ms : + clock->epoch_ms - (clock->monotonic_ms - mono_ms); +} + +void influx_clock_observe(influx_clock_t *clock, const char *date, uint64_t mono_ms) +{ + uint64_t server_ms; + if (!influx_parse_http_date(date, &server_ms)) return; + /* HTTP Date has one-second resolution. Use its midpoint; no external NTP needed. */ + if (!clock->ready) { + clock->epoch_ms = server_ms + 500; + clock->monotonic_ms = mono_ms; + clock->ready = true; + } else { + /* Correct oscillator drift gently; header rounding must not jitter every batch. */ + int64_t delta = (int64_t)(server_ms + 500) - (int64_t)clock_at(clock, mono_ms); + if (delta > 1500) clock->epoch_ms += 50; + else if (delta < -1500) clock->epoch_ms -= 50; + } +} + +bool influx_clock_timestamp(influx_clock_t *clock, uint64_t acquired_ms, uint64_t *timestamp) +{ + if (!clock->ready) return false; + uint64_t value = clock_at(clock, acquired_ms); + if (value <= clock->last_timestamp_ms) value = clock->last_timestamp_ms + 1; + clock->last_timestamp_ms = value; + *timestamp = value; + return true; +} + +bool influx_batch_add(influx_batch_t *batch, const char *line, uint64_t now_ms) +{ + size_t length = strlen(line), separator = batch->points ? 1 : 0; + if (!length || batch->points >= INFLUX_BATCH_POINTS || + length + separator >= sizeof(batch->body) - batch->bytes) return false; + if (!batch->points) batch->started_ms = now_ms; + if (separator) batch->body[batch->bytes++] = '\n'; + memcpy(batch->body + batch->bytes, line, length + 1); + batch->bytes += length; + const char *last_space = strrchr(line, ' '); + if (last_space) { + char *end; + uint64_t timestamp = strtoull(last_space + 1, &end, 10); + if (end != last_space + 1 && *end == 0) { + if (!batch->points) batch->first_timestamp_ms = timestamp; + batch->last_timestamp_ms = timestamp; + } + } + ++batch->points; + return true; +} + +bool influx_batch_due(const influx_batch_t *batch, uint64_t now_ms) +{ + return batch->points && (batch->points >= INFLUX_BATCH_POINTS || now_ms - batch->started_ms >= INFLUX_FLUSH_MS); +} diff --git a/firmware/components/net_influx/src/influx_batch.h b/firmware/components/net_influx/src/influx_batch.h new file mode 100644 index 0000000..c0073cc --- /dev/null +++ b/firmware/components/net_influx/src/influx_batch.h @@ -0,0 +1,27 @@ +#pragma once +#include +#include +#include + +#define INFLUX_LINE_CAP 512 +#define INFLUX_BATCH_CAP 4096 +#define INFLUX_BATCH_POINTS 10 +#define INFLUX_FLUSH_MS 10000 + +typedef struct { + char body[INFLUX_BATCH_CAP]; + size_t bytes; + unsigned points; + uint64_t started_ms; + uint64_t first_timestamp_ms, last_timestamp_ms; +} influx_batch_t; +typedef struct { + bool ready; + uint64_t epoch_ms, monotonic_ms, last_timestamp_ms; +} influx_clock_t; + +bool influx_parse_http_date(const char *date, uint64_t *epoch_ms); +void influx_clock_observe(influx_clock_t *clock, const char *date, uint64_t mono_ms); +bool influx_clock_timestamp(influx_clock_t *clock, uint64_t acquired_ms, uint64_t *timestamp); +bool influx_batch_add(influx_batch_t *batch, const char *line, uint64_t now_ms); +bool influx_batch_due(const influx_batch_t *batch, uint64_t now_ms); diff --git a/firmware/components/net_influx/src/net_influx.c b/firmware/components/net_influx/src/net_influx.c index d6fe696..ab21644 100644 --- a/firmware/components/net_influx/src/net_influx.c +++ b/firmware/components/net_influx/src/net_influx.c @@ -6,8 +6,13 @@ #include "freertos/FreeRTOS.h" #include "freertos/task.h" #include "freertos/queue.h" +#include "freertos/semphr.h" +#include "esp_timer.h" +#include "influx_batch.h" +#include #include +#include #include #include #include @@ -50,11 +55,11 @@ typedef struct { static QueueHandle_t s_q = NULL; static TaskHandle_t s_task = NULL; -static bool s_run = false; +static atomic_bool s_run = false; static net_influx_runtime_cfg_i_t s_rt = { - .queue_len = 16, - .task_stack = 6144, + .queue_len = 32, + .task_stack = 8192, .task_prio = 5, .init_retry_ms = 1000, .backoff_start_ms = 2000, @@ -131,28 +136,6 @@ static bool parse_base_url(const char* in, base_parts_t* out) { return true; } -static void mask_token(const char* token, char* out, size_t out_len) { - if (!token || !out || out_len < 8) return; - size_t n = strlen(token); - if (n <= 8) { - snprintf(out, out_len, "****"); - } else { - size_t keep = 4; - snprintf(out, out_len, "%.*s****%.*s", - (int)keep, token, - (int)keep, token + n - keep); - } -} - -static char* dup0(const char* str) { - if (!str) return NULL; - size_t n = strlen(str) + 1; - char* p = (char*)malloc(n); - if (!p) return NULL; - memcpy(p, str, n); - return p; -} - /* Minimal escaping für measurement/key: spaces/commas */ static size_t esc_copy(char* dst, size_t cap, const char* src, bool escape_equals) { size_t pos = 0; @@ -164,11 +147,11 @@ static size_t esc_copy(char* dst, size_t cap, const char* src, bool escape_equal char c = *p; bool esc = (c == ' ' || c == ',' || (escape_equals && c == '=')); if (esc) { - if (pos + 2 >= cap) break; + if (pos + 2 >= cap) { dst[0] = 0; return SIZE_MAX; } dst[pos++] = '\\'; dst[pos++] = c; } else { - if (pos + 1 >= cap) break; + if (pos + 1 >= cap) { dst[0] = 0; return SIZE_MAX; } dst[pos++] = c; } } @@ -176,123 +159,139 @@ static size_t esc_copy(char* dst, size_t cap, const char* src, bool escape_equal return pos; } -/* ---------------- HTTP Event Handler (IDF-kompatibel) ---------------- */ -static esp_err_t http_evt(esp_http_client_event_t* evt) { - switch (evt->event_id) { - case HTTP_EVENT_ERROR: ESP_LOGE(TAG, "HTTP_EVENT_ERROR"); break; - case HTTP_EVENT_ON_CONNECTED: ESP_LOGI(TAG, "HTTP_EVENT_ON_CONNECTED"); break; - case HTTP_EVENT_HEADER_SENT: ESP_LOGD(TAG, "HTTP_EVENT_HEADER_SENT"); break; - case HTTP_EVENT_ON_HEADER: - ESP_LOGD(TAG, "HDR: %s: %s", - evt->header_key ? evt->header_key : "(null)", - evt->header_value ? evt->header_value : "(null)"); - break; - case HTTP_EVENT_ON_DATA: ESP_LOGV(TAG, "DATA chunk len=%d", evt->data_len); break; - case HTTP_EVENT_ON_FINISH: ESP_LOGD(TAG, "HTTP_EVENT_ON_FINISH"); break; - case HTTP_EVENT_DISCONNECTED: ESP_LOGW(TAG, "HTTP_EVENT_DISCONNECTED"); break; - default: break; +/* Persistent HTTP client; shared with the optional synchronous API. */ +static esp_http_client_handle_t s_http; +static SemaphoreHandle_t s_http_lock; +static influx_clock_t s_clock; +static char s_write_url[640]; +static atomic_uint s_dropped; +static uint64_t monotonic_ms(void) { return (uint64_t)(esp_timer_get_time() / 1000); } + +static void http_close(void) +{ + if (s_http) esp_http_client_cleanup(s_http); + s_http = NULL; +} + +static esp_err_t http_evt(esp_http_client_event_t *event) +{ + if (event->event_id == HTTP_EVENT_ON_CONNECTED) ESP_LOGI(TAG, "HTTP connection established."); + if (event->event_id == HTTP_EVENT_ON_HEADER && event->header_key && event->header_value && + strcasecmp(event->header_key, "Date") == 0) { + bool ready = s_clock.ready; + influx_clock_observe(&s_clock, event->header_value, monotonic_ms()); + if (!ready && s_clock.ready) ESP_LOGI(TAG, "Measurement clock aligned to server HTTP Date."); } return ESP_OK; } -/* ---------------- Internal: HTTP send "line protocol" ---------------- */ -static esp_err_t net_influx_write_line(const char* line) { - bool ok = false; - - if (!line || !*line) { - sys_status_set_influx_ok(false); - return ESP_ERR_INVALID_ARG; +static bool encode_query(char *out, size_t cap, const char *text) +{ + static const char hex[] = "0123456789ABCDEF"; + size_t n = 0; + for (const unsigned char *p = (const unsigned char *)text; *p; ++p) { + bool plain = isalnum(*p) || *p == '-' || *p == '_' || *p == '.' || *p == '~'; + size_t need = plain ? 1 : 3; + if (n + need >= cap) return false; + if (plain) out[n++] = *p; + else { out[n++] = '%'; out[n++] = hex[*p >> 4]; out[n++] = hex[*p & 15]; } } - if (s.url_base[0] == 0 || s.org[0] == 0 || s.bucket[0] == 0 || s.token[0] == 0) { - sys_status_set_influx_ok(false); - return ESP_ERR_INVALID_STATE; - } - - // Base parse → host/port/tls - base_parts_t bp; - if (!parse_base_url(s.url_base, &bp)) { - ESP_LOGE(TAG, "Base-URL ungueltig: \"%s\"", s.url_base); - sys_status_set_influx_ok(false); - return ESP_ERR_INVALID_ARG; - } - if (s.use_tls) bp.tls = true; - int port = (bp.port > 0) ? bp.port : (bp.tls ? 443 : 8086); - - // Pfad - char path[192]; - snprintf(path, sizeof(path), "/api/v2/write?org=%s&bucket=%s&precision=ms", s.org, s.bucket); - - ESP_LOGI(TAG, "HTTP cfg: host=%s, port=%d, path=%s, tls=%d", bp.host, port, path, (int)bp.tls); - ESP_LOGD(TAG, "Line: %s", line); - - esp_http_client_config_t cfg = { - .host = bp.host, - .port = port, - .path = path, - .timeout_ms = s.http_timeout_ms, - .cert_pem = bp.tls ? s.ca_cert_pem : NULL, - .method = HTTP_METHOD_POST, - .event_handler = http_evt, - .transport_type = bp.tls ? HTTP_TRANSPORT_OVER_SSL : HTTP_TRANSPORT_OVER_TCP, - .disable_auto_redirect = false, - }; - - esp_http_client_handle_t h = esp_http_client_init(&cfg); - if (!h) { - ESP_LOGE(TAG, "http_client_init: no mem"); - sys_status_set_influx_ok(false); - return ESP_ERR_NO_MEM; - } - - // Header - size_t auth_len = strlen(s.token) + 7; // "Token " + '\0' - char* auth = (char*)malloc(auth_len); - if (!auth) { - ESP_LOGE(TAG, "malloc auth header: no mem"); - esp_http_client_cleanup(h); - sys_status_set_influx_ok(false); - return ESP_ERR_NO_MEM; - } - snprintf(auth, auth_len, "Token %s", s.token); - esp_http_client_set_header(h, "Authorization", auth); - free(auth); - - esp_http_client_set_header(h, "Content-Type", "text/plain; charset=utf-8"); - - esp_err_t err = esp_http_client_set_post_field(h, line, (int)strlen(line)); - if (err != ESP_OK) { - ESP_LOGE(TAG, "set_post_field: %s", esp_err_to_name(err)); - esp_http_client_cleanup(h); - sys_status_set_influx_ok(false); - return err; - } - - err = esp_http_client_perform(h); - - int status = -1; - if (err == ESP_OK) { - status = esp_http_client_get_status_code(h); - int64_t cl = esp_http_client_get_content_length(h); - ESP_LOGI(TAG, "HTTP status=%d, content_length=%lld", status, (long long)cl); - - if (status != 204) { - char resp[256] = {0}; - int n = esp_http_client_read_response(h, resp, sizeof(resp)-1); - if (n > 0) ESP_LOGW(TAG, "Resp(%d): %.*s", status, n, resp); - else ESP_LOGW(TAG, "Resp(%d): ", status); - } - } else { - ESP_LOGE(TAG, "perform: %s", esp_err_to_name(err)); - } - - esp_http_client_cleanup(h); - - ok = (err == ESP_OK && status == 204); - if (ok) sys_status_record_influx_success(); - sys_status_set_influx_ok(ok); - return ok ? ESP_OK : ESP_FAIL; + out[n] = 0; + return true; } +/* Caller holds s_http_lock. URLs/headers are configured once per connection. */ +static esp_err_t http_prepare(void) +{ + if (s_http) return ESP_OK; + if (!s.url_base[0] || !s.org[0] || !s.bucket[0] || !s.token[0]) return ESP_ERR_INVALID_STATE; + base_parts_t base; + if (!parse_base_url(s.url_base, &base)) return ESP_ERR_INVALID_ARG; + if (s.use_tls) base.tls = true; + int port = base.port > 0 ? base.port : base.tls ? 443 : 8086; + char org[192], bucket[192]; + if (!encode_query(org, sizeof(org), s.org) || !encode_query(bucket, sizeof(bucket), s.bucket)) return ESP_ERR_INVALID_SIZE; + int length = snprintf(s_write_url, sizeof(s_write_url), "%s://%s:%d/api/v2/write?org=%s&bucket=%s&precision=ms", + base.tls ? "https" : "http", base.host, port, org, bucket); + if (length < 0 || length >= sizeof(s_write_url)) return ESP_ERR_INVALID_SIZE; + esp_http_client_config_t cfg = { + .url = s_write_url, + .timeout_ms = s.http_timeout_ms, + .cert_pem = base.tls ? s.ca_cert_pem : NULL, + .method = HTTP_METHOD_POST, + .event_handler = http_evt, + .transport_type = base.tls ? HTTP_TRANSPORT_OVER_SSL : HTTP_TRANSPORT_OVER_TCP, + .disable_auto_redirect = true, + .keep_alive_enable = true, + .keep_alive_idle = 30, + .keep_alive_interval = 10, + .keep_alive_count = 3, + }; + s_http = esp_http_client_init(&cfg); + if (!s_http) return ESP_ERR_NO_MEM; + char auth[sizeof(s.token) + 7]; + snprintf(auth, sizeof(auth), "Token %s", s.token); + esp_err_t err = esp_http_client_set_header(s_http, "Authorization", auth); + if (err == ESP_OK) err = esp_http_client_set_header(s_http, "Content-Type", "text/plain; charset=utf-8"); + if (err != ESP_OK) { http_close(); return err; } + ESP_LOGI(TAG, "HTTP client ready / persistent connections enabled."); + return ESP_OK; +} + +/* One read-only request bootstraps the private measurement clock, even without Internet/NTP. */ +static void clock_bootstrap(void) +{ + xSemaphoreTake(s_http_lock, portMAX_DELAY); + if (!s_clock.ready && http_prepare() == ESP_OK) { + char health_url[160]; + base_parts_t base; + if (parse_base_url(s.url_base, &base)) { + if (s.use_tls) base.tls = true; + snprintf(health_url, sizeof(health_url), "%s://%s:%d/health", base.tls ? "https" : "http", + base.host, base.port > 0 ? base.port : base.tls ? 443 : 8086); + esp_http_client_set_url(s_http, health_url); + esp_http_client_set_method(s_http, HTTP_METHOD_GET); + esp_http_client_set_post_field(s_http, NULL, 0); + esp_err_t err = esp_http_client_perform(s_http); + if (err != ESP_OK) http_close(); + else { + esp_http_client_set_url(s_http, s_write_url); + esp_http_client_set_method(s_http, HTTP_METHOD_POST); + } + } + if (!s_clock.ready) ESP_LOGW(TAG, "No server Date available; safe single-point fallback."); + } + xSemaphoreGive(s_http_lock); +} + +static esp_err_t net_influx_write_payload(const char *body, unsigned points, int *status_out) +{ + if (status_out) *status_out = 0; + if (!body || !*body || !s_http_lock) return ESP_ERR_INVALID_ARG; + xSemaphoreTake(s_http_lock, portMAX_DELAY); + int status = 0; + esp_err_t err = http_prepare(); + if (err == ESP_OK) err = esp_http_client_set_post_field(s_http, body, (int)strlen(body)); + if (err == ESP_OK) err = esp_http_client_perform(s_http); + if (err == ESP_OK) status = esp_http_client_get_status_code(s_http); + bool ok = err == ESP_OK && status == 204; + if (ok) { + ESP_LOGI(TAG, "HTTP status=204, points=%u, bytes=%u", points, (unsigned)strlen(body)); + sys_status_record_influx_success(); + } else { + ESP_LOGW(TAG, "Write failed: HTTP=%d error=%s points=%u", status, esp_err_to_name(err), points); + http_close(); + } + /* Do not retain a pointer to the caller's temporary line/batch buffer. */ + if (s_http) esp_http_client_set_post_field(s_http, NULL, 0); + sys_status_set_influx_ok(ok); + if (status_out) *status_out = status; + xSemaphoreGive(s_http_lock); + return ok ? ESP_OK : err != ESP_OK ? err : ESP_FAIL; +} + +static esp_err_t net_influx_write_line(const char *line) { return net_influx_write_payload(line, 1, NULL); } + /* ---------------- Generic Point/Fields → Line Protocol ---------------- */ static bool field_is_valid(const net_influx_field_t* f) { if (!f || !f->key || !*f->key) return false; @@ -306,11 +305,18 @@ static esp_err_t build_line_protocol(const net_influx_point_t* p, char* out, siz return ESP_ERR_INVALID_ARG; } + if (strpbrk(p->measurement, "\r\n") || (p->tags && strpbrk(p->tags, "\r\n"))) return ESP_ERR_INVALID_ARG; + for (size_t i = 0; i < p->num_fields; ++i) { + const net_influx_field_t *field = &p->fields[i]; + if ((field->key && strpbrk(field->key, "\r\n")) || + (field->type == NET_INFLUX_F_STRING && field->v.s && strpbrk(field->v.s, "\r\n"))) + return ESP_ERR_INVALID_ARG; + } size_t off = 0; // measurement escaped (space/comma) char meas[96]; - esc_copy(meas, sizeof(meas), p->measurement, false); + if (esc_copy(meas, sizeof(meas), p->measurement, false) == SIZE_MAX) return ESP_ERR_INVALID_SIZE; off += snprintf(out + off, out_cap - off, "%s", meas); if (off >= out_cap) return ESP_FAIL; @@ -330,7 +336,7 @@ static esp_err_t build_line_protocol(const net_influx_point_t* p, char* out, siz if (!field_is_valid(f)) continue; char key[64]; - esc_copy(key, sizeof(key), f->key, false); + if (esc_copy(key, sizeof(key), f->key, true) == SIZE_MAX) return ESP_ERR_INVALID_SIZE; if (!first) { off += snprintf(out + off, out_cap - off, ","); @@ -350,7 +356,7 @@ static esp_err_t build_line_protocol(const net_influx_point_t* p, char* out, siz off += snprintf(out + off, out_cap - off, "%s=%lldi", key, (long long)f->v.i); break; case NET_INFLUX_F_UINT: - off += snprintf(out + off, out_cap - off, "%s=%llui", key, (unsigned long long)f->v.u); + off += snprintf(out + off, out_cap - off, "%s=%lluu", key, (unsigned long long)f->v.u); break; case NET_INFLUX_F_BOOL: off += snprintf(out + off, out_cap - off, "%s=%s", key, f->v.b ? "true" : "false"); @@ -387,124 +393,116 @@ static esp_err_t build_line_protocol(const net_influx_point_t* p, char* out, siz return ESP_OK; } -/* ---------------- Async Queue Message (deep copy) ---------------- */ +/* Fixed-size, pre-serialized queue entries: no per-field heap allocation. */ typedef struct { - char* measurement; - char* tags; // optional - uint64_t ts_ms; - - size_t num_fields; - net_influx_field_t* fields; // deep-copied array; keys and string-values are duplicated + char line[INFLUX_LINE_CAP]; + uint64_t acquired_ms; + bool timestamped; } influx_msg_t; -static void free_msg(influx_msg_t* m) { - if (!m) return; - if (m->measurement) free(m->measurement); - if (m->tags) free(m->tags); - if (m->fields) { - for (size_t i = 0; i < m->num_fields; ++i) { - if (m->fields[i].type == NET_INFLUX_F_STRING && m->fields[i].v.s) { - free((void*)m->fields[i].v.s); - } - if (m->fields[i].key) free((void*)m->fields[i].key); - } - free(m->fields); +static bool wait_running(uint32_t ms) +{ + while (s_run && ms) { + uint32_t step = ms > 100 ? 100 : ms; + vTaskDelay(pdMS_TO_TICKS(step)); + ms -= step; } - memset(m, 0, sizeof(*m)); + return s_run; } -/* ---------------- Init/Backoff (im Worker) ---------------- */ -static bool influx_init_blocking(void) { +static bool influx_init_blocking(void) +{ while (s_run) { - if (net_influx_init_from_nvs(s_rt.ca_cert_pem_opt, s_rt.http_timeout_ms) == ESP_OK) { - ESP_LOGI(TAG, "Influx ready (from NVS)."); - return true; - } - ESP_LOGW(TAG, "Influx nicht bereit – retry in %u ms", (unsigned)s_rt.init_retry_ms); - vTaskDelay(pdMS_TO_TICKS(s_rt.init_retry_ms)); + if (net_influx_init_from_nvs(s_rt.ca_cert_pem_opt, s_rt.http_timeout_ms) == ESP_OK) return true; + ESP_LOGW(TAG, "Influx configuration unavailable; retrying."); + wait_running(s_rt.init_retry_ms); } return false; } -static bool influx_reinit_with_backoff(uint32_t* backoff_ms) { - if (!backoff_ms) return false; - - uint32_t wait_ms = (*backoff_ms == 0) ? s_rt.backoff_start_ms : *backoff_ms; - ESP_LOGW(TAG, "Re-Init Influx in %u ms", (unsigned)wait_ms); - vTaskDelay(pdMS_TO_TICKS(wait_ms)); - - bool ok = (net_influx_init_from_nvs(s_rt.ca_cert_pem_opt, s_rt.http_timeout_ms) == ESP_OK); - if (ok) { - ESP_LOGI(TAG, "Influx Re-Init OK"); - *backoff_ms = s_rt.backoff_start_ms; - return true; - } - - if (wait_ms < s_rt.backoff_max_ms) { - uint32_t next = wait_ms * 2; - *backoff_ms = (next > s_rt.backoff_max_ms) ? s_rt.backoff_max_ms : next; - } - return false; -} - -/* ---------------- Worker Task ---------------- */ -static void net_influx_worker(void* arg) { +static void net_influx_worker(void *arg) +{ (void)arg; - ESP_LOGI(TAG, "Influx worker started."); - - if (!influx_init_blocking()) { - ESP_LOGE(TAG, "Worker beendet (Influx init fatal)."); - vTaskDelete(NULL); - return; - } - - int fail_count = 0; - uint32_t backoff_ms = s_rt.backoff_start_ms; - + influx_batch_t *batch = calloc(1, sizeof(*batch)); + influx_msg_t carry = {0}; + bool have_carry = false, batch_timestamped = true; + unsigned failures = 0; + uint32_t backoff = s_rt.backoff_start_ms; + if (!batch) { ESP_LOGE(TAG, "Cannot allocate Influx batch buffer."); goto done; } + if (!influx_init_blocking()) goto done; + clock_bootstrap(); + ESP_LOGI(TAG, "Influx batching ready: up to %u points / %u ms, queue=%u", INFLUX_BATCH_POINTS, INFLUX_FLUSH_MS, s_rt.queue_len); while (s_run) { - influx_msg_t msg = {0}; - if (xQueueReceive(s_q, &msg, portMAX_DELAY) != pdTRUE) continue; - - net_influx_point_t p = { - .measurement = msg.measurement, - .tags = msg.tags, - .ts_ms = msg.ts_ms, - .fields = msg.fields, - .num_fields = msg.num_fields, - }; - - char line[512]; - esp_err_t b = build_line_protocol(&p, line, sizeof(line)); - - if (b == ESP_ERR_INVALID_STATE) { - ESP_LOGW(TAG, "Skipping write: no valid fields."); - free_msg(&msg); - continue; + bool flush = influx_batch_due(batch, monotonic_ms()); + if (!flush) { + influx_msg_t msg; + bool received = have_carry; + if (received) { msg = carry; have_carry = false; } + else received = xQueueReceive(s_q, &msg, pdMS_TO_TICKS(100)) == pdTRUE; + if (!received) continue; + if (!msg.timestamped) { + uint64_t timestamp; + xSemaphoreTake(s_http_lock, portMAX_DELAY); + bool known = influx_clock_timestamp(&s_clock, msg.acquired_ms, ×tamp); + xSemaphoreGive(s_http_lock); + if (known) { + size_t used = strlen(msg.line); + int n = snprintf(msg.line + used, sizeof(msg.line) - used, " %llu", (unsigned long long)timestamp); + if (n < 0 || (size_t)n >= sizeof(msg.line) - used) { ESP_LOGW(TAG, "Dropping oversized timestamped point."); continue; } + msg.timestamped = true; + } + } + if ((!msg.timestamped && batch->points) || !influx_batch_add(batch, msg.line, monotonic_ms())) { + carry = msg; have_carry = true; flush = true; + } else { + batch_timestamped = msg.timestamped; + flush = !msg.timestamped || influx_batch_due(batch, monotonic_ms()); + } } - if (b != ESP_OK) { - ESP_LOGW(TAG, "Line build failed: %s", esp_err_to_name(b)); - free_msg(&msg); - continue; - } - - esp_err_t e = net_influx_write_line(line); - - if (e == ESP_OK) { - if (fail_count) ESP_LOGI(TAG, "Influx write recovered."); - fail_count = 0; - backoff_ms = s_rt.backoff_start_ms; - } else { - ESP_LOGW(TAG, "Influx write failed: %s", esp_err_to_name(e)); - if (++fail_count >= (int)s_rt.fails_before_reinit) { - (void)influx_reinit_with_backoff(&backoff_ms); - fail_count = 0; + if (!flush || !batch->points) continue; + /* Retries keep the exact body AND timestamps; acknowledged points are never resubmitted. */ + while (s_run && batch->points) { + int status; + esp_err_t err = net_influx_write_payload(batch->body, batch->points, &status); + if (err == ESP_OK) { + ESP_LOGI(TAG, "Batch UTC: first_ms=%llu last_ms=%llu dropped=%u", + (unsigned long long)batch->first_timestamp_ms, + (unsigned long long)batch->last_timestamp_ms, + (unsigned)atomic_load(&s_dropped)); + memset(batch, 0, sizeof(*batch)); + failures = 0; backoff = s_rt.backoff_start_ms; + break; + } + if (!batch_timestamped || status == 400 || status == 413 || status == 422) { + if (!batch_timestamped) + ESP_LOGW(TAG, "Single-point request failed without a clock; no automatic replay with a new server timestamp."); + else + ESP_LOGW(TAG, "Dropping %u rejected points (HTTP %d); inspect schema/payload limits.", batch->points, status); + memset(batch, 0, sizeof(*batch)); + break; + } + ++failures; + ESP_LOGW(TAG, "Retaining %u points; retry in %u ms.", batch->points, (unsigned)backoff); + if (!wait_running(backoff)) break; + if (failures >= s_rt.fails_before_reinit) { + net_influx_init_from_nvs(s_rt.ca_cert_pem_opt, s_rt.http_timeout_ms); + failures = 0; + } + if (backoff < s_rt.backoff_max_ms) { + uint64_t next = (uint64_t)backoff * 2; + backoff = next > s_rt.backoff_max_ms ? s_rt.backoff_max_ms : (uint32_t)next; } } - - free_msg(&msg); } - - ESP_LOGI(TAG, "Influx worker stopping."); +done: + if (batch && batch->points) ESP_LOGW(TAG, "Stopping: %u pending points discarded from RAM.", batch->points); + free(batch); + if (s_http_lock) { + xSemaphoreTake(s_http_lock, portMAX_DELAY); http_close(); xSemaphoreGive(s_http_lock); + } + if (s_q) { vQueueDelete(s_q); s_q = NULL; } + s_run = false; + s_task = NULL; vTaskDelete(NULL); } @@ -513,20 +511,26 @@ esp_err_t net_influx_init(const net_influx_cfg_t* cfg) { if (!cfg || !cfg->url_base || !cfg->org || !cfg->bucket || !cfg->token) { return ESP_ERR_INVALID_ARG; } - safe_copy(s.url_base, sizeof(s.url_base), cfg->url_base); - safe_copy(s.org, sizeof(s.org), cfg->org); - safe_copy(s.bucket, sizeof(s.bucket), cfg->bucket); - safe_copy(s.token, sizeof(s.token), cfg->token); - trim_inplace(s.url_base); - - s.use_tls = cfg->use_tls; - s.ca_cert_pem = cfg->ca_cert_pem; - s.http_timeout_ms = (cfg->http_timeout_ms > 0) ? cfg->http_timeout_ms : 3000; - - char tok[32] = {0}; - mask_token(s.token, tok, sizeof(tok)); - ESP_LOGI(TAG, "init: base=\"%s\", org=\"%s\", bucket=\"%s\", tls=%d, timeout=%dms, token=%s", - s.url_base, s.org, s.bucket, (int)s.use_tls, s.http_timeout_ms, tok); + if (!s_http_lock) { + s_http_lock = xSemaphoreCreateMutex(); + if (!s_http_lock) return ESP_ERR_NO_MEM; + } + net_influx_state_t next = {0}; + safe_copy(next.url_base, sizeof(next.url_base), cfg->url_base); + safe_copy(next.org, sizeof(next.org), cfg->org); + safe_copy(next.bucket, sizeof(next.bucket), cfg->bucket); + safe_copy(next.token, sizeof(next.token), cfg->token); + trim_inplace(next.url_base); + next.use_tls = cfg->use_tls; + next.ca_cert_pem = cfg->ca_cert_pem; + next.http_timeout_ms = cfg->http_timeout_ms > 0 ? cfg->http_timeout_ms : 3000; + xSemaphoreTake(s_http_lock, portMAX_DELAY); + bool host_changed = strcmp(s.url_base, next.url_base) || s.use_tls != next.use_tls; + if (memcmp(&s, &next, sizeof(s))) http_close(); + if (host_changed) memset(&s_clock, 0, sizeof(s_clock)); + s = next; + xSemaphoreGive(s_http_lock); + ESP_LOGI(TAG, "Influx configuration loaded; batching uses existing organization/bucket."); sys_status_set_influx_ok(false); return ESP_OK; } @@ -553,8 +557,9 @@ esp_err_t net_influx_write_fields(const char* measurement, char fields[256]; va_list ap; va_start(ap, fields_fmt); - vsnprintf(fields, sizeof(fields), fields_fmt, ap); + int fields_length = vsnprintf(fields, sizeof(fields), fields_fmt, ap); va_end(ap); + if (fields_length < 0 || (size_t)fields_length >= sizeof(fields)) return ESP_ERR_INVALID_SIZE; // 2) Line Protocol char line[512]; @@ -562,7 +567,7 @@ esp_err_t net_influx_write_fields(const char* measurement, // measurement escaping (space/comma) char meas[96]; - esc_copy(meas, sizeof(meas), measurement, false); + if (esc_copy(meas, sizeof(meas), measurement, false) == SIZE_MAX) return ESP_ERR_INVALID_SIZE; off += snprintf(line + off, sizeof(line) - off, "%s", meas); if (off < 0 || off >= (int)sizeof(line)) { sys_status_set_influx_ok(false); return ESP_FAIL; } @@ -620,7 +625,7 @@ esp_err_t net_influx_init_from_nvs(const char* ca_cert_pem_opt, int timeout_ms) /* ---------------- NEW: Start/Stop/Submit (generic, multi-sensor) ---------------- */ esp_err_t net_influx_start(const net_influx_runtime_cfg_t* rt) { - if (s_task) return ESP_OK; + if (s_task) return s_run ? ESP_OK : ESP_ERR_INVALID_STATE; // runtime cfg übernehmen (mit Defaults) if (rt) { @@ -640,6 +645,7 @@ esp_err_t net_influx_start(const net_influx_runtime_cfg_t* rt) { s_q = xQueueCreate(s_rt.queue_len, sizeof(influx_msg_t)); if (!s_q) return ESP_ERR_NO_MEM; + s_dropped = 0; s_run = true; if (xTaskCreate(net_influx_worker, "influx_worker", s_rt.task_stack, NULL, s_rt.task_prio, &s_task) != pdPASS) { vQueueDelete(s_q); @@ -650,71 +656,32 @@ esp_err_t net_influx_start(const net_influx_runtime_cfg_t* rt) { return ESP_OK; } -void net_influx_stop(void) { +void net_influx_stop(void) +{ s_run = false; - - if (s_task) { - vTaskDelete(s_task); - s_task = NULL; - } - if (s_q) { - influx_msg_t msg; - while (xQueueReceive(s_q, &msg, 0) == pdTRUE) { - free_msg(&msg); - } - vQueueDelete(s_q); - s_q = NULL; - } + /* Worker owns its HTTP handle, batch and queue. Never delete it mid-request. */ + for (unsigned i = 0; s_task && i < 50; ++i) vTaskDelay(pdMS_TO_TICKS(100)); + if (s_task) ESP_LOGW(TAG, "Worker still finishing an HTTP request; cleanup deferred."); } -esp_err_t net_influx_submit_point(const net_influx_point_t* p) { - if (!p || !p->measurement || !*p->measurement || !p->fields || p->num_fields == 0) { +esp_err_t net_influx_submit_point(const net_influx_point_t *point) +{ + if (!point || !point->measurement || !*point->measurement || !point->fields || !point->num_fields || point->ts_ms > 9223372036854ULL) return ESP_ERR_INVALID_ARG; - } - if (!s_q) return ESP_ERR_INVALID_STATE; - - influx_msg_t msg = {0}; - msg.measurement = dup0(p->measurement); - msg.tags = (p->tags && *p->tags) ? dup0(p->tags) : NULL; - msg.ts_ms = p->ts_ms; - msg.num_fields = p->num_fields; - - msg.fields = (net_influx_field_t*)calloc(p->num_fields, sizeof(net_influx_field_t)); - if (!msg.measurement || !msg.fields) { - free_msg(&msg); - return ESP_ERR_NO_MEM; - } - - for (size_t i = 0; i < p->num_fields; ++i) { - msg.fields[i] = p->fields[i]; - - // deep copy key - msg.fields[i].key = dup0(p->fields[i].key); - if (!msg.fields[i].key) { - free_msg(&msg); - return ESP_ERR_NO_MEM; - } - - // deep copy string value if needed - if (p->fields[i].type == NET_INFLUX_F_STRING && p->fields[i].v.s) { - msg.fields[i].v.s = dup0(p->fields[i].v.s); - if (!msg.fields[i].v.s) { - free_msg(&msg); - return ESP_ERR_NO_MEM; - } - } - } - - // Non-blocking enqueue; wenn voll: drop oldest + if (!s_run || !s_q) return ESP_ERR_INVALID_STATE; + influx_msg_t msg = {.acquired_ms = monotonic_ms(), .timestamped = point->ts_ms != 0}; + /* Reserve room for an automatically assigned millisecond timestamp. */ + size_t cap = msg.timestamped ? sizeof(msg.line) : sizeof(msg.line) - 24; + esp_err_t err = build_line_protocol(point, msg.line, cap); + if (err != ESP_OK) return err; if (xQueueSend(s_q, &msg, 0) != pdPASS) { - influx_msg_t dump; - if (xQueueReceive(s_q, &dump, 0) == pdTRUE) free_msg(&dump); - - if (xQueueSend(s_q, &msg, 0) != pdPASS) { - free_msg(&msg); - return ESP_FAIL; + influx_msg_t old; + if (xQueueReceive(s_q, &old, 0) == pdTRUE) { + unsigned dropped = atomic_fetch_add(&s_dropped, 1) + 1; + if (dropped == 1 || dropped % 10 == 0) + ESP_LOGW(TAG, "Influx queue full: %u old pending points dropped since start.", dropped); } + if (xQueueSend(s_q, &msg, 0) != pdPASS) return ESP_FAIL; } - return ESP_OK; -} \ No newline at end of file +} diff --git a/firmware/components/sen5x/air_sensor_app.c b/firmware/components/sen5x/air_sensor_app.c index b236275..c73fcef 100644 --- a/firmware/components/sen5x/air_sensor_app.c +++ b/firmware/components/sen5x/air_sensor_app.c @@ -34,8 +34,8 @@ static const char* TAG = "air_sensor_app"; #define AIR_SENSOR_PROD_PRIO 6 // Producer leicht höher priorisiert // Influx worker runtime (läuft in net_influx.c) -#define INFLUX_QUEUE_LEN 16 -#define INFLUX_TASK_STACK 6144 +#define INFLUX_QUEUE_LEN 32 +#define INFLUX_TASK_STACK 8192 #define INFLUX_TASK_PRIO 5 #define INFLUX_INIT_RETRY_MS 1000 @@ -254,7 +254,7 @@ static void air_sensor_task_producer(void* arg) { net_influx_point_t p = { .measurement = "air", .tags = g_device_tag ? g_device_tag : "device=unknown", - .ts_ms = 0, // Serverzeit + .ts_ms = 0, // Messzeitpunkt; Worker richtet UTC an Server-HTTP-Date aus .fields = f, .num_fields = sizeof(f) / sizeof(f[0]), }; diff --git a/firmware/test/net_influx/fake_sdk.h b/firmware/test/net_influx/fake_sdk.h new file mode 100644 index 0000000..73de491 --- /dev/null +++ b/firmware/test/net_influx/fake_sdk.h @@ -0,0 +1,72 @@ +#pragma once +#include +#include +#include +#include +typedef int esp_err_t; +#define ESP_OK 0 +#define ESP_FAIL -1 +#define ESP_ERR_INVALID_ARG 1 +#define ESP_ERR_NO_MEM 2 +#define ESP_ERR_INVALID_STATE 3 +#define ESP_ERR_NOT_FOUND 4 +#define ESP_ERR_INVALID_SIZE 5 +const char *esp_err_to_name(esp_err_t code); +void fake_log(const char *level, const char *tag, const char *format, ...); +#define ESP_LOGI(tag, ...) fake_log("I", tag, __VA_ARGS__) +#define ESP_LOGW(tag, ...) fake_log("W", tag, __VA_ARGS__) +#define ESP_LOGE(tag, ...) fake_log("E", tag, __VA_ARGS__) +#define ESP_LOGD(tag, ...) fake_log("D", tag, __VA_ARGS__) +#define ESP_LOGV(tag, ...) fake_log("V", tag, __VA_ARGS__) +typedef unsigned UBaseType_t; +typedef int BaseType_t; +typedef uint32_t TickType_t; +typedef void *TaskHandle_t; +typedef struct fake_queue *QueueHandle_t; +typedef void *SemaphoreHandle_t; +#define pdTRUE 1 +#define pdPASS 1 +#define portMAX_DELAY UINT32_MAX +#define pdMS_TO_TICKS(ms) (ms) +QueueHandle_t xQueueCreate(unsigned count, unsigned size); +int xQueueSend(QueueHandle_t queue, const void *item, unsigned ticks); +int xQueueReceive(QueueHandle_t queue, void *item, unsigned ticks); +void vQueueDelete(QueueHandle_t queue); +int xTaskCreate(void (*task)(void *), const char *name, unsigned stack, void *arg, unsigned prio, TaskHandle_t *out); +void vTaskDelay(unsigned ticks); +void vTaskDelete(TaskHandle_t task); +SemaphoreHandle_t xSemaphoreCreateMutex(void); +int xSemaphoreTake(SemaphoreHandle_t mutex, unsigned ticks); +int xSemaphoreGive(SemaphoreHandle_t mutex); +int64_t esp_timer_get_time(void); +void sys_status_set_influx_ok(bool value); +void sys_status_record_influx_success(void); +bool storage_load_service_url(char **url); +bool storage_get_effective_influx(char **org, char **bucket, char **token); +typedef struct fake_http *esp_http_client_handle_t; +typedef struct { + int event_id; + char *header_key, *header_value; +} esp_http_client_event_t; +#define HTTP_EVENT_ON_HEADER 1 +#define HTTP_EVENT_ON_CONNECTED 2 +#define HTTP_METHOD_POST 1 +#define HTTP_METHOD_GET 2 +#define HTTP_TRANSPORT_OVER_SSL 1 +#define HTTP_TRANSPORT_OVER_TCP 2 +typedef struct { + const char *url; + int timeout_ms, method, transport_type; + const char *cert_pem; + esp_err_t (*event_handler)(esp_http_client_event_t *); + bool disable_auto_redirect, keep_alive_enable; + int keep_alive_idle, keep_alive_interval, keep_alive_count; +} esp_http_client_config_t; +esp_http_client_handle_t esp_http_client_init(const esp_http_client_config_t *config); +esp_err_t esp_http_client_cleanup(esp_http_client_handle_t client); +esp_err_t esp_http_client_set_header(esp_http_client_handle_t client, const char *key, const char *value); +esp_err_t esp_http_client_set_url(esp_http_client_handle_t client, const char *url); +esp_err_t esp_http_client_set_method(esp_http_client_handle_t client, int method); +esp_err_t esp_http_client_set_post_field(esp_http_client_handle_t client, const char *data, int length); +esp_err_t esp_http_client_perform(esp_http_client_handle_t client); +int esp_http_client_get_status_code(esp_http_client_handle_t client); diff --git a/firmware/test/net_influx/run_tests.py b/firmware/test/net_influx/run_tests.py new file mode 100644 index 0000000..a7678d1 --- /dev/null +++ b/firmware/test/net_influx/run_tests.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Compile the real Influx worker against a deterministic FreeRTOS/HTTP simulator.""" +from pathlib import Path +import subprocess +import tempfile + +firmware = Path(__file__).resolve().parents[2] +tests = Path(__file__).resolve().parent +source = firmware / 'components/net_influx' +with tempfile.TemporaryDirectory(prefix='nanopsx-influx-test-') as directory: + build = Path(directory) + for header in ['esp_err.h', 'esp_log.h', 'esp_timer.h', 'esp_http_client.h', + 'storage.h', 'sys_status.h', 'freertos/FreeRTOS.h', + 'freertos/task.h', 'freertos/queue.h', 'freertos/semphr.h']: + path = build / header + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('#include "fake_sdk.h"\n') + program = build / 'test_runtime' + subprocess.run(['cc', '-std=c11', '-D_POSIX_C_SOURCE=200809L', '-Wall', '-Wextra', + '-Wno-sign-compare', '-g', '-fsanitize=address,undefined', + '-fno-omit-frame-pointer', '-fno-pie', '-no-pie', + '-I', str(build), '-I', str(tests), '-I', str(source / 'include'), + '-I', str(source / 'src'), str(tests / 'test_runtime.c'), + str(source / 'src/influx_batch.c'), '-lm', '-o', str(program)], check=True) + for scenario in ['format', 'clock', 'normal', 'retry', 'overflow', 'no-date', + 'no-date-failure', 'partial', 'max-body', 'reject']: + subprocess.run([str(program), scenario], check=True) diff --git a/firmware/test/net_influx/test_runtime.c b/firmware/test/net_influx/test_runtime.c new file mode 100644 index 0000000..9bf38c3 --- /dev/null +++ b/firmware/test/net_influx/test_runtime.c @@ -0,0 +1,204 @@ +/* The production worker, formatter, queue submission and client lifetime are exercised below. */ +#include "fake_sdk.h" +#include +#include +#include +#include +#include +#include "net_influx.c" + +static const char *scenario; +static uint64_t now_ms, next_sample_ms=1000; +static unsigned produced, target=30, posts, gets, clients, cleanups, accepted, rejected, successes; +static unsigned failures_left, samples_per_post[128], seen[128], max_queue; +static uint64_t stamp[128]; +static bool status_ok; +static char failed_body[INFLUX_BATCH_CAP]; +static void (*task_entry)(void *); +struct fake_queue { unsigned cap, size, count, head; unsigned char *data; }; +struct fake_http { esp_http_client_config_t cfg; char url[640]; const char *body; int length, method, status; }; + +const char *esp_err_to_name(esp_err_t code) { return code==0?"ESP_OK":"fake error"; } +void fake_log(const char *level,const char *tag,const char *fmt,...) { (void)level;(void)tag;(void)fmt; } +int64_t esp_timer_get_time(void) { return now_ms*1000; } +void sys_status_set_influx_ok(bool value) { status_ok=value; } +void sys_status_record_influx_success(void) { ++successes; } +SemaphoreHandle_t xSemaphoreCreateMutex(void) { return (void *)1; } +int xSemaphoreTake(SemaphoreHandle_t mutex,unsigned ticks) { (void)ticks;assert(mutex);return pdTRUE; } +int xSemaphoreGive(SemaphoreHandle_t mutex) { assert(mutex);return pdTRUE; } +bool storage_load_service_url(char **value) { *value=strdup("http://influx.test:8086");return true; } +bool storage_get_effective_influx(char **org,char **bucket,char **token) +{ *org=strdup("Home");*bucket=strdup("nanoPSX");*token=strdup("synthetic-test-token");return true; } +QueueHandle_t xQueueCreate(unsigned count,unsigned size) +{ + struct fake_queue *q=calloc(1,sizeof(*q));assert(q); + q->cap=count;q->size=size;q->data=calloc(count,size);assert(q->data);return q; +} +int xQueueSend(QueueHandle_t q,const void *data,unsigned ticks) +{ + (void)ticks;if(q->count==q->cap)return 0; + memcpy(q->data+((q->head+q->count)%q->cap)*q->size,data,q->size);++q->count; + if(q->count>max_queue)max_queue=q->count; + return pdPASS; +} +static void generate(void) +{ + ++produced; + char detail[421];memset(detail,'a',sizeof(detail)-1);detail[sizeof(detail)-1]=0; + net_influx_field_t fields[]={ + {.key="value",.type=NET_INFLUX_F_INT,.v.i=produced}, + {.key="detail",.type=NET_INFLUX_F_STRING,.v.s=detail} + }; + net_influx_point_t point={.measurement="air",.tags="device=test",.fields=fields, + .num_fields=!strcmp(scenario,"max-body")?2:1}; + assert(net_influx_submit_point(&point)==ESP_OK); + fields[0].v.i=-1;memset(detail,'X',sizeof(detail)-1); /* Queue must own serialized bytes. */ +} +static void advance(uint64_t destination) +{ + while(producedcount && ticks) { + uint64_t destination=now_ms+ticks; + if(producedcount)return 0; + memcpy(data,q->data+q->head*q->size,q->size);q->head=(q->head+1)%q->cap;--q->count;return pdTRUE; +} +void vQueueDelete(QueueHandle_t q) { free(q->data);free(q); } +int xTaskCreate(void (*fn)(void *),const char *name,unsigned stack,void *arg,unsigned prio,TaskHandle_t *out) +{ (void)name;(void)arg;(void)prio;assert(stack>=8192);task_entry=fn;*out=(void *)1;return pdPASS; } +void vTaskDelay(unsigned ticks) { advance(now_ms+ticks); } +void vTaskDelete(TaskHandle_t task) { assert(!task); } + +esp_http_client_handle_t esp_http_client_init(const esp_http_client_config_t *cfg) +{ + struct fake_http *h=calloc(1,sizeof(*h));assert(h); + h->cfg=*cfg;h->method=cfg->method;snprintf(h->url,sizeof(h->url),"%s",cfg->url); + assert(cfg->keep_alive_enable && cfg->disable_auto_redirect);++clients; + esp_http_client_event_t event={.event_id=HTTP_EVENT_ON_CONNECTED};cfg->event_handler(&event);return h; +} +esp_err_t esp_http_client_cleanup(esp_http_client_handle_t h) { ++cleanups;free(h);return ESP_OK; } +esp_err_t esp_http_client_set_header(esp_http_client_handle_t h,const char *key,const char *value) +{ (void)h;assert(key && value);return ESP_OK; } +esp_err_t esp_http_client_set_url(esp_http_client_handle_t h,const char *url) +{ snprintf(h->url,sizeof(h->url),"%s",url);return ESP_OK; } +esp_err_t esp_http_client_set_method(esp_http_client_handle_t h,int method) { h->method=method;return ESP_OK; } +esp_err_t esp_http_client_set_post_field(esp_http_client_handle_t h,const char *body,int length) +{ h->body=body;h->length=length;return ESP_OK; } +int esp_http_client_get_status_code(esp_http_client_handle_t h) { return h->status; } +static bool no_date(void) { return !strncmp(scenario,"no-date",7); } +static void date_event(esp_http_client_handle_t h) +{ + if(no_date())return; + uint64_t base;assert(influx_parse_http_date("Wed, 11 Nov 2026 12:00:00 GMT",&base)); + time_t seconds=(time_t)((base+now_ms)/1000); + char date[40];strftime(date,sizeof(date),"%a, %d %b %Y %H:%M:%S GMT",gmtime(&seconds)); + esp_http_client_event_t event={.event_id=HTTP_EVENT_ON_HEADER,.header_key="dAtE",.header_value=date}; + h->cfg.event_handler(&event); +} +esp_err_t esp_http_client_perform(esp_http_client_handle_t h) +{ + date_event(h); + if(h->method==HTTP_METHOD_GET) { assert(strstr(h->url,"/health"));++gets;h->status=200;return ESP_OK; } + assert(strstr(h->url,"/api/v2/write?org=Home&bucket=nanoPSX&precision=ms")); + assert(h->body && h->length>0 && h->lengthbody;*p;++p)if(*p=='\n')++count; + assert(count<=INFLUX_BATCH_POINTS && posts<128);samples_per_post[posts++]=count; + if(failures_left) { + if(!failed_body[0])snprintf(failed_body,sizeof(failed_body),"%s",h->body); + else assert(!strcmp(failed_body,h->body)); + --failures_left;return ESP_FAIL; + } + if(failed_body[0]) { assert(!strcmp(failed_body,h->body));failed_body[0]=0; } + if(!strcmp(scenario,"no-date-failure") && posts==1) { ++rejected;return ESP_FAIL; } + if(!strcmp(scenario,"reject") && posts==1) { rejected+=count;h->status=400;return ESP_OK; } + char payload[INFLUX_BATCH_CAP];snprintf(payload,sizeof(payload),"%s",h->body); + for(char *line=strtok(payload,"\n");line;line=strtok(NULL,"\n")) { + unsigned id;assert(sscanf(line,"air,device=test value=%ui",&id)==1 && id>0 && id<=target); + assert(!seen[id]++); + const char *last=strrchr(line,' ');assert(last); + if(no_date()) { assert(count==1 && !strcmp(last+1,line+16)); } + else { + char *end;uint64_t timestamp=strtoull(last+1,&end,10);assert(*end==0); + uint64_t base;assert(influx_parse_http_date("Wed, 11 Nov 2026 12:00:00 GMT",&base)); + assert(timestamp==base+500+id*1000);stamp[id]=timestamp; + } + if(!strcmp(scenario,"max-body")) { assert(strstr(line,"detail=\"aaaa"));assert(!strstr(line,"XXXX")); } + } + accepted+=count;h->status=204; + if(accepted+rejected+atomic_load(&s_dropped)>=target)s_run=false; + return ESP_OK; +} + +static void test_format(void) +{ + char line[512]; + net_influx_field_t fields[]={ + {.key="temp",.type=NET_INFLUX_F_FLOAT,.precision=1,.v.f=23.4}, + {.key="missing",.type=NET_INFLUX_F_FLOAT,.v.f=NAN}, + {.key="a=b",.type=NET_INFLUX_F_UINT,.v.u=42}, + {.key="flag",.type=NET_INFLUX_F_BOOL,.v.b=true}, + {.key="text",.type=NET_INFLUX_F_STRING,.v.s="a\"b\\c"} + }; + net_influx_point_t p={.measurement="air room",.tags="device=test",.ts_ms=123456,.fields=fields,.num_fields=5}; + assert(build_line_protocol(&p,line,sizeof(line))==ESP_OK); + assert(!strcmp(line,"air\\ room,device=test temp=23.4,a\\=b=42u,flag=true,text=\"a\\\"b\\\\c\" 123456")); + fields[4].v.s="newline\nnot allowed";assert(build_line_protocol(&p,line,sizeof(line))==ESP_ERR_INVALID_ARG); + p.num_fields=1;fields[0].v.f=NAN;assert(build_line_protocol(&p,line,sizeof(line))==ESP_ERR_INVALID_STATE); + fields[0].v.f=1; + char large[200];memset(large,'a',sizeof(large)-1);large[199]=0;p.measurement=large; + assert(build_line_protocol(&p,line,sizeof(line))==ESP_ERR_INVALID_SIZE); +} +static void test_clock(void) +{ + uint64_t epoch; + assert(influx_parse_http_date("Wed, 11 Nov 2026 12:00:00 GMT",&epoch)); + assert(epoch==1794398400000ULL); + assert(!influx_parse_http_date("Thu, 29 Feb 2100 00:00:00 GMT",&epoch)); + assert(!influx_parse_http_date("Thu, 31 Feb 2024 00:00:00 GMT",&epoch)); + assert(!influx_parse_http_date("Wed, 11 Nov 2026 12:00:00 UTC",&epoch)); + assert(influx_parse_http_date("Thu, 29 Feb 2024 00:00:00 GMT",&epoch)); + influx_clock_t clock={0};uint64_t first,second; + assert(!influx_clock_timestamp(&clock,0,&first)); + influx_clock_observe(&clock,"Wed, 11 Nov 2026 12:00:00 GMT",1000); + assert(influx_clock_timestamp(&clock,500,&first)); + assert(influx_clock_timestamp(&clock,1500,&second) && second-first==1000); + influx_batch_t b={0};assert(!influx_batch_due(&b,0)); + assert(influx_batch_add(&b,"air value=1i 12345",50)); + assert(!influx_batch_due(&b,10049) && influx_batch_due(&b,10050)); + assert(b.first_timestamp_ms==12345 && b.last_timestamp_ms==12345); + for(int i=1;i<10;++i)assert(influx_batch_add(&b,"air value=2i 12346",100)); + assert(influx_batch_due(&b,100) && !influx_batch_add(&b,"air value=3i 12347",100)); +} +int main(int argc,char **argv) +{ + assert(argc==2);scenario=argv[1]; + if(!strcmp(scenario,"format"))test_format(); + else if(!strcmp(scenario,"clock"))test_clock(); + else { + if(!strcmp(scenario,"retry"))failures_left=1; + if(!strcmp(scenario,"overflow")) { failures_left=5;target=70; } + if(!strcmp(scenario,"partial"))target=3; + if(!strcmp(scenario,"max-body"))target=10; + assert(net_influx_start(NULL)==ESP_OK);assert(task_entry);task_entry(NULL); + assert(!s_task && !s_q && !s_http && clients==cleanups); + assert(accepted+rejected+atomic_load(&s_dropped)==target && max_queue<=32); + if(!strcmp(scenario,"normal"))assert(posts==3 && accepted==30 && clients==1 && gets==1 && successes==3); + if(!strcmp(scenario,"retry"))assert(posts==4 && clients==2 && accepted==30); + if(!strcmp(scenario,"overflow"))assert(atomic_load(&s_dropped)>0 && max_queue==32 && accepted>=32); + if(!strcmp(scenario,"partial"))assert(posts==1 && samples_per_post[0]==3 && now_ms==11000); + if(!strcmp(scenario,"max-body"))assert(posts==2 && accepted==10); + if(!strcmp(scenario,"reject"))assert(posts==3 && accepted==20 && rejected==10); + if(no_date())assert(posts==target && clients<=2); + assert(status_ok); + } + printf("PASS: Influx %s\n",scenario); +}