perf(influx): batch measurements and reuse HTTP connections

This commit is contained in:
root
2026-11-11 22:57:12 +00:00
parent dde4736cc2
commit e88623cf6e
11 changed files with 782 additions and 325 deletions
+72
View File
@@ -0,0 +1,72 @@
#pragma once
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <stdarg.h>
typedef int esp_err_t;
#define ESP_OK 0
#define ESP_FAIL -1
#define ESP_ERR_INVALID_ARG 1
#define ESP_ERR_NO_MEM 2
#define ESP_ERR_INVALID_STATE 3
#define ESP_ERR_NOT_FOUND 4
#define ESP_ERR_INVALID_SIZE 5
const char *esp_err_to_name(esp_err_t code);
void fake_log(const char *level, const char *tag, const char *format, ...);
#define ESP_LOGI(tag, ...) fake_log("I", tag, __VA_ARGS__)
#define ESP_LOGW(tag, ...) fake_log("W", tag, __VA_ARGS__)
#define ESP_LOGE(tag, ...) fake_log("E", tag, __VA_ARGS__)
#define ESP_LOGD(tag, ...) fake_log("D", tag, __VA_ARGS__)
#define ESP_LOGV(tag, ...) fake_log("V", tag, __VA_ARGS__)
typedef unsigned UBaseType_t;
typedef int BaseType_t;
typedef uint32_t TickType_t;
typedef void *TaskHandle_t;
typedef struct fake_queue *QueueHandle_t;
typedef void *SemaphoreHandle_t;
#define pdTRUE 1
#define pdPASS 1
#define portMAX_DELAY UINT32_MAX
#define pdMS_TO_TICKS(ms) (ms)
QueueHandle_t xQueueCreate(unsigned count, unsigned size);
int xQueueSend(QueueHandle_t queue, const void *item, unsigned ticks);
int xQueueReceive(QueueHandle_t queue, void *item, unsigned ticks);
void vQueueDelete(QueueHandle_t queue);
int xTaskCreate(void (*task)(void *), const char *name, unsigned stack, void *arg, unsigned prio, TaskHandle_t *out);
void vTaskDelay(unsigned ticks);
void vTaskDelete(TaskHandle_t task);
SemaphoreHandle_t xSemaphoreCreateMutex(void);
int xSemaphoreTake(SemaphoreHandle_t mutex, unsigned ticks);
int xSemaphoreGive(SemaphoreHandle_t mutex);
int64_t esp_timer_get_time(void);
void sys_status_set_influx_ok(bool value);
void sys_status_record_influx_success(void);
bool storage_load_service_url(char **url);
bool storage_get_effective_influx(char **org, char **bucket, char **token);
typedef struct fake_http *esp_http_client_handle_t;
typedef struct {
int event_id;
char *header_key, *header_value;
} esp_http_client_event_t;
#define HTTP_EVENT_ON_HEADER 1
#define HTTP_EVENT_ON_CONNECTED 2
#define HTTP_METHOD_POST 1
#define HTTP_METHOD_GET 2
#define HTTP_TRANSPORT_OVER_SSL 1
#define HTTP_TRANSPORT_OVER_TCP 2
typedef struct {
const char *url;
int timeout_ms, method, transport_type;
const char *cert_pem;
esp_err_t (*event_handler)(esp_http_client_event_t *);
bool disable_auto_redirect, keep_alive_enable;
int keep_alive_idle, keep_alive_interval, keep_alive_count;
} esp_http_client_config_t;
esp_http_client_handle_t esp_http_client_init(const esp_http_client_config_t *config);
esp_err_t esp_http_client_cleanup(esp_http_client_handle_t client);
esp_err_t esp_http_client_set_header(esp_http_client_handle_t client, const char *key, const char *value);
esp_err_t esp_http_client_set_url(esp_http_client_handle_t client, const char *url);
esp_err_t esp_http_client_set_method(esp_http_client_handle_t client, int method);
esp_err_t esp_http_client_set_post_field(esp_http_client_handle_t client, const char *data, int length);
esp_err_t esp_http_client_perform(esp_http_client_handle_t client);
int esp_http_client_get_status_code(esp_http_client_handle_t client);
+27
View File
@@ -0,0 +1,27 @@
#!/usr/bin/env python3
"""Compile the real Influx worker against a deterministic FreeRTOS/HTTP simulator."""
from pathlib import Path
import subprocess
import tempfile
firmware = Path(__file__).resolve().parents[2]
tests = Path(__file__).resolve().parent
source = firmware / 'components/net_influx'
with tempfile.TemporaryDirectory(prefix='nanopsx-influx-test-') as directory:
build = Path(directory)
for header in ['esp_err.h', 'esp_log.h', 'esp_timer.h', 'esp_http_client.h',
'storage.h', 'sys_status.h', 'freertos/FreeRTOS.h',
'freertos/task.h', 'freertos/queue.h', 'freertos/semphr.h']:
path = build / header
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text('#include "fake_sdk.h"\n')
program = build / 'test_runtime'
subprocess.run(['cc', '-std=c11', '-D_POSIX_C_SOURCE=200809L', '-Wall', '-Wextra',
'-Wno-sign-compare', '-g', '-fsanitize=address,undefined',
'-fno-omit-frame-pointer', '-fno-pie', '-no-pie',
'-I', str(build), '-I', str(tests), '-I', str(source / 'include'),
'-I', str(source / 'src'), str(tests / 'test_runtime.c'),
str(source / 'src/influx_batch.c'), '-lm', '-o', str(program)], check=True)
for scenario in ['format', 'clock', 'normal', 'retry', 'overflow', 'no-date',
'no-date-failure', 'partial', 'max-body', 'reject']:
subprocess.run([str(program), scenario], check=True)
+204
View File
@@ -0,0 +1,204 @@
/* The production worker, formatter, queue submission and client lifetime are exercised below. */
#include "fake_sdk.h"
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "net_influx.c"
static const char *scenario;
static uint64_t now_ms, next_sample_ms=1000;
static unsigned produced, target=30, posts, gets, clients, cleanups, accepted, rejected, successes;
static unsigned failures_left, samples_per_post[128], seen[128], max_queue;
static uint64_t stamp[128];
static bool status_ok;
static char failed_body[INFLUX_BATCH_CAP];
static void (*task_entry)(void *);
struct fake_queue { unsigned cap, size, count, head; unsigned char *data; };
struct fake_http { esp_http_client_config_t cfg; char url[640]; const char *body; int length, method, status; };
const char *esp_err_to_name(esp_err_t code) { return code==0?"ESP_OK":"fake error"; }
void fake_log(const char *level,const char *tag,const char *fmt,...) { (void)level;(void)tag;(void)fmt; }
int64_t esp_timer_get_time(void) { return now_ms*1000; }
void sys_status_set_influx_ok(bool value) { status_ok=value; }
void sys_status_record_influx_success(void) { ++successes; }
SemaphoreHandle_t xSemaphoreCreateMutex(void) { return (void *)1; }
int xSemaphoreTake(SemaphoreHandle_t mutex,unsigned ticks) { (void)ticks;assert(mutex);return pdTRUE; }
int xSemaphoreGive(SemaphoreHandle_t mutex) { assert(mutex);return pdTRUE; }
bool storage_load_service_url(char **value) { *value=strdup("http://influx.test:8086");return true; }
bool storage_get_effective_influx(char **org,char **bucket,char **token)
{ *org=strdup("Home");*bucket=strdup("nanoPSX");*token=strdup("synthetic-test-token");return true; }
QueueHandle_t xQueueCreate(unsigned count,unsigned size)
{
struct fake_queue *q=calloc(1,sizeof(*q));assert(q);
q->cap=count;q->size=size;q->data=calloc(count,size);assert(q->data);return q;
}
int xQueueSend(QueueHandle_t q,const void *data,unsigned ticks)
{
(void)ticks;if(q->count==q->cap)return 0;
memcpy(q->data+((q->head+q->count)%q->cap)*q->size,data,q->size);++q->count;
if(q->count>max_queue)max_queue=q->count;
return pdPASS;
}
static void generate(void)
{
++produced;
char detail[421];memset(detail,'a',sizeof(detail)-1);detail[sizeof(detail)-1]=0;
net_influx_field_t fields[]={
{.key="value",.type=NET_INFLUX_F_INT,.v.i=produced},
{.key="detail",.type=NET_INFLUX_F_STRING,.v.s=detail}
};
net_influx_point_t point={.measurement="air",.tags="device=test",.fields=fields,
.num_fields=!strcmp(scenario,"max-body")?2:1};
assert(net_influx_submit_point(&point)==ESP_OK);
fields[0].v.i=-1;memset(detail,'X',sizeof(detail)-1); /* Queue must own serialized bytes. */
}
static void advance(uint64_t destination)
{
while(produced<target && next_sample_ms<=destination) {
now_ms=next_sample_ms;next_sample_ms+=1000;generate();
}
now_ms=destination;
assert(now_ms<1000000); /* Detect a stuck retry/flush loop. */
}
int xQueueReceive(QueueHandle_t q,void *data,unsigned ticks)
{
if(!q->count && ticks) {
uint64_t destination=now_ms+ticks;
if(produced<target && next_sample_ms<destination)destination=next_sample_ms;
advance(destination);
}
if(!q->count)return 0;
memcpy(data,q->data+q->head*q->size,q->size);q->head=(q->head+1)%q->cap;--q->count;return pdTRUE;
}
void vQueueDelete(QueueHandle_t q) { free(q->data);free(q); }
int xTaskCreate(void (*fn)(void *),const char *name,unsigned stack,void *arg,unsigned prio,TaskHandle_t *out)
{ (void)name;(void)arg;(void)prio;assert(stack>=8192);task_entry=fn;*out=(void *)1;return pdPASS; }
void vTaskDelay(unsigned ticks) { advance(now_ms+ticks); }
void vTaskDelete(TaskHandle_t task) { assert(!task); }
esp_http_client_handle_t esp_http_client_init(const esp_http_client_config_t *cfg)
{
struct fake_http *h=calloc(1,sizeof(*h));assert(h);
h->cfg=*cfg;h->method=cfg->method;snprintf(h->url,sizeof(h->url),"%s",cfg->url);
assert(cfg->keep_alive_enable && cfg->disable_auto_redirect);++clients;
esp_http_client_event_t event={.event_id=HTTP_EVENT_ON_CONNECTED};cfg->event_handler(&event);return h;
}
esp_err_t esp_http_client_cleanup(esp_http_client_handle_t h) { ++cleanups;free(h);return ESP_OK; }
esp_err_t esp_http_client_set_header(esp_http_client_handle_t h,const char *key,const char *value)
{ (void)h;assert(key && value);return ESP_OK; }
esp_err_t esp_http_client_set_url(esp_http_client_handle_t h,const char *url)
{ snprintf(h->url,sizeof(h->url),"%s",url);return ESP_OK; }
esp_err_t esp_http_client_set_method(esp_http_client_handle_t h,int method) { h->method=method;return ESP_OK; }
esp_err_t esp_http_client_set_post_field(esp_http_client_handle_t h,const char *body,int length)
{ h->body=body;h->length=length;return ESP_OK; }
int esp_http_client_get_status_code(esp_http_client_handle_t h) { return h->status; }
static bool no_date(void) { return !strncmp(scenario,"no-date",7); }
static void date_event(esp_http_client_handle_t h)
{
if(no_date())return;
uint64_t base;assert(influx_parse_http_date("Wed, 11 Nov 2026 12:00:00 GMT",&base));
time_t seconds=(time_t)((base+now_ms)/1000);
char date[40];strftime(date,sizeof(date),"%a, %d %b %Y %H:%M:%S GMT",gmtime(&seconds));
esp_http_client_event_t event={.event_id=HTTP_EVENT_ON_HEADER,.header_key="dAtE",.header_value=date};
h->cfg.event_handler(&event);
}
esp_err_t esp_http_client_perform(esp_http_client_handle_t h)
{
date_event(h);
if(h->method==HTTP_METHOD_GET) { assert(strstr(h->url,"/health"));++gets;h->status=200;return ESP_OK; }
assert(strstr(h->url,"/api/v2/write?org=Home&bucket=nanoPSX&precision=ms"));
assert(h->body && h->length>0 && h->length<INFLUX_BATCH_CAP);
unsigned count=1;for(const char *p=h->body;*p;++p)if(*p=='\n')++count;
assert(count<=INFLUX_BATCH_POINTS && posts<128);samples_per_post[posts++]=count;
if(failures_left) {
if(!failed_body[0])snprintf(failed_body,sizeof(failed_body),"%s",h->body);
else assert(!strcmp(failed_body,h->body));
--failures_left;return ESP_FAIL;
}
if(failed_body[0]) { assert(!strcmp(failed_body,h->body));failed_body[0]=0; }
if(!strcmp(scenario,"no-date-failure") && posts==1) { ++rejected;return ESP_FAIL; }
if(!strcmp(scenario,"reject") && posts==1) { rejected+=count;h->status=400;return ESP_OK; }
char payload[INFLUX_BATCH_CAP];snprintf(payload,sizeof(payload),"%s",h->body);
for(char *line=strtok(payload,"\n");line;line=strtok(NULL,"\n")) {
unsigned id;assert(sscanf(line,"air,device=test value=%ui",&id)==1 && id>0 && id<=target);
assert(!seen[id]++);
const char *last=strrchr(line,' ');assert(last);
if(no_date()) { assert(count==1 && !strcmp(last+1,line+16)); }
else {
char *end;uint64_t timestamp=strtoull(last+1,&end,10);assert(*end==0);
uint64_t base;assert(influx_parse_http_date("Wed, 11 Nov 2026 12:00:00 GMT",&base));
assert(timestamp==base+500+id*1000);stamp[id]=timestamp;
}
if(!strcmp(scenario,"max-body")) { assert(strstr(line,"detail=\"aaaa"));assert(!strstr(line,"XXXX")); }
}
accepted+=count;h->status=204;
if(accepted+rejected+atomic_load(&s_dropped)>=target)s_run=false;
return ESP_OK;
}
static void test_format(void)
{
char line[512];
net_influx_field_t fields[]={
{.key="temp",.type=NET_INFLUX_F_FLOAT,.precision=1,.v.f=23.4},
{.key="missing",.type=NET_INFLUX_F_FLOAT,.v.f=NAN},
{.key="a=b",.type=NET_INFLUX_F_UINT,.v.u=42},
{.key="flag",.type=NET_INFLUX_F_BOOL,.v.b=true},
{.key="text",.type=NET_INFLUX_F_STRING,.v.s="a\"b\\c"}
};
net_influx_point_t p={.measurement="air room",.tags="device=test",.ts_ms=123456,.fields=fields,.num_fields=5};
assert(build_line_protocol(&p,line,sizeof(line))==ESP_OK);
assert(!strcmp(line,"air\\ room,device=test temp=23.4,a\\=b=42u,flag=true,text=\"a\\\"b\\\\c\" 123456"));
fields[4].v.s="newline\nnot allowed";assert(build_line_protocol(&p,line,sizeof(line))==ESP_ERR_INVALID_ARG);
p.num_fields=1;fields[0].v.f=NAN;assert(build_line_protocol(&p,line,sizeof(line))==ESP_ERR_INVALID_STATE);
fields[0].v.f=1;
char large[200];memset(large,'a',sizeof(large)-1);large[199]=0;p.measurement=large;
assert(build_line_protocol(&p,line,sizeof(line))==ESP_ERR_INVALID_SIZE);
}
static void test_clock(void)
{
uint64_t epoch;
assert(influx_parse_http_date("Wed, 11 Nov 2026 12:00:00 GMT",&epoch));
assert(epoch==1794398400000ULL);
assert(!influx_parse_http_date("Thu, 29 Feb 2100 00:00:00 GMT",&epoch));
assert(!influx_parse_http_date("Thu, 31 Feb 2024 00:00:00 GMT",&epoch));
assert(!influx_parse_http_date("Wed, 11 Nov 2026 12:00:00 UTC",&epoch));
assert(influx_parse_http_date("Thu, 29 Feb 2024 00:00:00 GMT",&epoch));
influx_clock_t clock={0};uint64_t first,second;
assert(!influx_clock_timestamp(&clock,0,&first));
influx_clock_observe(&clock,"Wed, 11 Nov 2026 12:00:00 GMT",1000);
assert(influx_clock_timestamp(&clock,500,&first));
assert(influx_clock_timestamp(&clock,1500,&second) && second-first==1000);
influx_batch_t b={0};assert(!influx_batch_due(&b,0));
assert(influx_batch_add(&b,"air value=1i 12345",50));
assert(!influx_batch_due(&b,10049) && influx_batch_due(&b,10050));
assert(b.first_timestamp_ms==12345 && b.last_timestamp_ms==12345);
for(int i=1;i<10;++i)assert(influx_batch_add(&b,"air value=2i 12346",100));
assert(influx_batch_due(&b,100) && !influx_batch_add(&b,"air value=3i 12347",100));
}
int main(int argc,char **argv)
{
assert(argc==2);scenario=argv[1];
if(!strcmp(scenario,"format"))test_format();
else if(!strcmp(scenario,"clock"))test_clock();
else {
if(!strcmp(scenario,"retry"))failures_left=1;
if(!strcmp(scenario,"overflow")) { failures_left=5;target=70; }
if(!strcmp(scenario,"partial"))target=3;
if(!strcmp(scenario,"max-body"))target=10;
assert(net_influx_start(NULL)==ESP_OK);assert(task_entry);task_entry(NULL);
assert(!s_task && !s_q && !s_http && clients==cleanups);
assert(accepted+rejected+atomic_load(&s_dropped)==target && max_queue<=32);
if(!strcmp(scenario,"normal"))assert(posts==3 && accepted==30 && clients==1 && gets==1 && successes==3);
if(!strcmp(scenario,"retry"))assert(posts==4 && clients==2 && accepted==30);
if(!strcmp(scenario,"overflow"))assert(atomic_load(&s_dropped)>0 && max_queue==32 && accepted>=32);
if(!strcmp(scenario,"partial"))assert(posts==1 && samples_per_post[0]==3 && now_ms==11000);
if(!strcmp(scenario,"max-body"))assert(posts==2 && accepted==10);
if(!strcmp(scenario,"reject"))assert(posts==3 && accepted==20 && rejected==10);
if(no_date())assert(posts==target && clients<=2);
assert(status_ok);
}
printf("PASS: Influx %s\n",scenario);
}