perf(influx): batch measurements and reuse HTTP connections
This commit is contained in:
@@ -0,0 +1,204 @@
|
||||
/* The production worker, formatter, queue submission and client lifetime are exercised below. */
|
||||
#include "fake_sdk.h"
|
||||
#include <assert.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
#include "net_influx.c"
|
||||
|
||||
static const char *scenario;
|
||||
static uint64_t now_ms, next_sample_ms=1000;
|
||||
static unsigned produced, target=30, posts, gets, clients, cleanups, accepted, rejected, successes;
|
||||
static unsigned failures_left, samples_per_post[128], seen[128], max_queue;
|
||||
static uint64_t stamp[128];
|
||||
static bool status_ok;
|
||||
static char failed_body[INFLUX_BATCH_CAP];
|
||||
static void (*task_entry)(void *);
|
||||
struct fake_queue { unsigned cap, size, count, head; unsigned char *data; };
|
||||
struct fake_http { esp_http_client_config_t cfg; char url[640]; const char *body; int length, method, status; };
|
||||
|
||||
const char *esp_err_to_name(esp_err_t code) { return code==0?"ESP_OK":"fake error"; }
|
||||
void fake_log(const char *level,const char *tag,const char *fmt,...) { (void)level;(void)tag;(void)fmt; }
|
||||
int64_t esp_timer_get_time(void) { return now_ms*1000; }
|
||||
void sys_status_set_influx_ok(bool value) { status_ok=value; }
|
||||
void sys_status_record_influx_success(void) { ++successes; }
|
||||
SemaphoreHandle_t xSemaphoreCreateMutex(void) { return (void *)1; }
|
||||
int xSemaphoreTake(SemaphoreHandle_t mutex,unsigned ticks) { (void)ticks;assert(mutex);return pdTRUE; }
|
||||
int xSemaphoreGive(SemaphoreHandle_t mutex) { assert(mutex);return pdTRUE; }
|
||||
bool storage_load_service_url(char **value) { *value=strdup("http://influx.test:8086");return true; }
|
||||
bool storage_get_effective_influx(char **org,char **bucket,char **token)
|
||||
{ *org=strdup("Home");*bucket=strdup("nanoPSX");*token=strdup("synthetic-test-token");return true; }
|
||||
QueueHandle_t xQueueCreate(unsigned count,unsigned size)
|
||||
{
|
||||
struct fake_queue *q=calloc(1,sizeof(*q));assert(q);
|
||||
q->cap=count;q->size=size;q->data=calloc(count,size);assert(q->data);return q;
|
||||
}
|
||||
int xQueueSend(QueueHandle_t q,const void *data,unsigned ticks)
|
||||
{
|
||||
(void)ticks;if(q->count==q->cap)return 0;
|
||||
memcpy(q->data+((q->head+q->count)%q->cap)*q->size,data,q->size);++q->count;
|
||||
if(q->count>max_queue)max_queue=q->count;
|
||||
return pdPASS;
|
||||
}
|
||||
static void generate(void)
|
||||
{
|
||||
++produced;
|
||||
char detail[421];memset(detail,'a',sizeof(detail)-1);detail[sizeof(detail)-1]=0;
|
||||
net_influx_field_t fields[]={
|
||||
{.key="value",.type=NET_INFLUX_F_INT,.v.i=produced},
|
||||
{.key="detail",.type=NET_INFLUX_F_STRING,.v.s=detail}
|
||||
};
|
||||
net_influx_point_t point={.measurement="air",.tags="device=test",.fields=fields,
|
||||
.num_fields=!strcmp(scenario,"max-body")?2:1};
|
||||
assert(net_influx_submit_point(&point)==ESP_OK);
|
||||
fields[0].v.i=-1;memset(detail,'X',sizeof(detail)-1); /* Queue must own serialized bytes. */
|
||||
}
|
||||
static void advance(uint64_t destination)
|
||||
{
|
||||
while(produced<target && next_sample_ms<=destination) {
|
||||
now_ms=next_sample_ms;next_sample_ms+=1000;generate();
|
||||
}
|
||||
now_ms=destination;
|
||||
assert(now_ms<1000000); /* Detect a stuck retry/flush loop. */
|
||||
}
|
||||
int xQueueReceive(QueueHandle_t q,void *data,unsigned ticks)
|
||||
{
|
||||
if(!q->count && ticks) {
|
||||
uint64_t destination=now_ms+ticks;
|
||||
if(produced<target && next_sample_ms<destination)destination=next_sample_ms;
|
||||
advance(destination);
|
||||
}
|
||||
if(!q->count)return 0;
|
||||
memcpy(data,q->data+q->head*q->size,q->size);q->head=(q->head+1)%q->cap;--q->count;return pdTRUE;
|
||||
}
|
||||
void vQueueDelete(QueueHandle_t q) { free(q->data);free(q); }
|
||||
int xTaskCreate(void (*fn)(void *),const char *name,unsigned stack,void *arg,unsigned prio,TaskHandle_t *out)
|
||||
{ (void)name;(void)arg;(void)prio;assert(stack>=8192);task_entry=fn;*out=(void *)1;return pdPASS; }
|
||||
void vTaskDelay(unsigned ticks) { advance(now_ms+ticks); }
|
||||
void vTaskDelete(TaskHandle_t task) { assert(!task); }
|
||||
|
||||
esp_http_client_handle_t esp_http_client_init(const esp_http_client_config_t *cfg)
|
||||
{
|
||||
struct fake_http *h=calloc(1,sizeof(*h));assert(h);
|
||||
h->cfg=*cfg;h->method=cfg->method;snprintf(h->url,sizeof(h->url),"%s",cfg->url);
|
||||
assert(cfg->keep_alive_enable && cfg->disable_auto_redirect);++clients;
|
||||
esp_http_client_event_t event={.event_id=HTTP_EVENT_ON_CONNECTED};cfg->event_handler(&event);return h;
|
||||
}
|
||||
esp_err_t esp_http_client_cleanup(esp_http_client_handle_t h) { ++cleanups;free(h);return ESP_OK; }
|
||||
esp_err_t esp_http_client_set_header(esp_http_client_handle_t h,const char *key,const char *value)
|
||||
{ (void)h;assert(key && value);return ESP_OK; }
|
||||
esp_err_t esp_http_client_set_url(esp_http_client_handle_t h,const char *url)
|
||||
{ snprintf(h->url,sizeof(h->url),"%s",url);return ESP_OK; }
|
||||
esp_err_t esp_http_client_set_method(esp_http_client_handle_t h,int method) { h->method=method;return ESP_OK; }
|
||||
esp_err_t esp_http_client_set_post_field(esp_http_client_handle_t h,const char *body,int length)
|
||||
{ h->body=body;h->length=length;return ESP_OK; }
|
||||
int esp_http_client_get_status_code(esp_http_client_handle_t h) { return h->status; }
|
||||
static bool no_date(void) { return !strncmp(scenario,"no-date",7); }
|
||||
static void date_event(esp_http_client_handle_t h)
|
||||
{
|
||||
if(no_date())return;
|
||||
uint64_t base;assert(influx_parse_http_date("Wed, 11 Nov 2026 12:00:00 GMT",&base));
|
||||
time_t seconds=(time_t)((base+now_ms)/1000);
|
||||
char date[40];strftime(date,sizeof(date),"%a, %d %b %Y %H:%M:%S GMT",gmtime(&seconds));
|
||||
esp_http_client_event_t event={.event_id=HTTP_EVENT_ON_HEADER,.header_key="dAtE",.header_value=date};
|
||||
h->cfg.event_handler(&event);
|
||||
}
|
||||
esp_err_t esp_http_client_perform(esp_http_client_handle_t h)
|
||||
{
|
||||
date_event(h);
|
||||
if(h->method==HTTP_METHOD_GET) { assert(strstr(h->url,"/health"));++gets;h->status=200;return ESP_OK; }
|
||||
assert(strstr(h->url,"/api/v2/write?org=Home&bucket=nanoPSX&precision=ms"));
|
||||
assert(h->body && h->length>0 && h->length<INFLUX_BATCH_CAP);
|
||||
unsigned count=1;for(const char *p=h->body;*p;++p)if(*p=='\n')++count;
|
||||
assert(count<=INFLUX_BATCH_POINTS && posts<128);samples_per_post[posts++]=count;
|
||||
if(failures_left) {
|
||||
if(!failed_body[0])snprintf(failed_body,sizeof(failed_body),"%s",h->body);
|
||||
else assert(!strcmp(failed_body,h->body));
|
||||
--failures_left;return ESP_FAIL;
|
||||
}
|
||||
if(failed_body[0]) { assert(!strcmp(failed_body,h->body));failed_body[0]=0; }
|
||||
if(!strcmp(scenario,"no-date-failure") && posts==1) { ++rejected;return ESP_FAIL; }
|
||||
if(!strcmp(scenario,"reject") && posts==1) { rejected+=count;h->status=400;return ESP_OK; }
|
||||
char payload[INFLUX_BATCH_CAP];snprintf(payload,sizeof(payload),"%s",h->body);
|
||||
for(char *line=strtok(payload,"\n");line;line=strtok(NULL,"\n")) {
|
||||
unsigned id;assert(sscanf(line,"air,device=test value=%ui",&id)==1 && id>0 && id<=target);
|
||||
assert(!seen[id]++);
|
||||
const char *last=strrchr(line,' ');assert(last);
|
||||
if(no_date()) { assert(count==1 && !strcmp(last+1,line+16)); }
|
||||
else {
|
||||
char *end;uint64_t timestamp=strtoull(last+1,&end,10);assert(*end==0);
|
||||
uint64_t base;assert(influx_parse_http_date("Wed, 11 Nov 2026 12:00:00 GMT",&base));
|
||||
assert(timestamp==base+500+id*1000);stamp[id]=timestamp;
|
||||
}
|
||||
if(!strcmp(scenario,"max-body")) { assert(strstr(line,"detail=\"aaaa"));assert(!strstr(line,"XXXX")); }
|
||||
}
|
||||
accepted+=count;h->status=204;
|
||||
if(accepted+rejected+atomic_load(&s_dropped)>=target)s_run=false;
|
||||
return ESP_OK;
|
||||
}
|
||||
|
||||
static void test_format(void)
|
||||
{
|
||||
char line[512];
|
||||
net_influx_field_t fields[]={
|
||||
{.key="temp",.type=NET_INFLUX_F_FLOAT,.precision=1,.v.f=23.4},
|
||||
{.key="missing",.type=NET_INFLUX_F_FLOAT,.v.f=NAN},
|
||||
{.key="a=b",.type=NET_INFLUX_F_UINT,.v.u=42},
|
||||
{.key="flag",.type=NET_INFLUX_F_BOOL,.v.b=true},
|
||||
{.key="text",.type=NET_INFLUX_F_STRING,.v.s="a\"b\\c"}
|
||||
};
|
||||
net_influx_point_t p={.measurement="air room",.tags="device=test",.ts_ms=123456,.fields=fields,.num_fields=5};
|
||||
assert(build_line_protocol(&p,line,sizeof(line))==ESP_OK);
|
||||
assert(!strcmp(line,"air\\ room,device=test temp=23.4,a\\=b=42u,flag=true,text=\"a\\\"b\\\\c\" 123456"));
|
||||
fields[4].v.s="newline\nnot allowed";assert(build_line_protocol(&p,line,sizeof(line))==ESP_ERR_INVALID_ARG);
|
||||
p.num_fields=1;fields[0].v.f=NAN;assert(build_line_protocol(&p,line,sizeof(line))==ESP_ERR_INVALID_STATE);
|
||||
fields[0].v.f=1;
|
||||
char large[200];memset(large,'a',sizeof(large)-1);large[199]=0;p.measurement=large;
|
||||
assert(build_line_protocol(&p,line,sizeof(line))==ESP_ERR_INVALID_SIZE);
|
||||
}
|
||||
static void test_clock(void)
|
||||
{
|
||||
uint64_t epoch;
|
||||
assert(influx_parse_http_date("Wed, 11 Nov 2026 12:00:00 GMT",&epoch));
|
||||
assert(epoch==1794398400000ULL);
|
||||
assert(!influx_parse_http_date("Thu, 29 Feb 2100 00:00:00 GMT",&epoch));
|
||||
assert(!influx_parse_http_date("Thu, 31 Feb 2024 00:00:00 GMT",&epoch));
|
||||
assert(!influx_parse_http_date("Wed, 11 Nov 2026 12:00:00 UTC",&epoch));
|
||||
assert(influx_parse_http_date("Thu, 29 Feb 2024 00:00:00 GMT",&epoch));
|
||||
influx_clock_t clock={0};uint64_t first,second;
|
||||
assert(!influx_clock_timestamp(&clock,0,&first));
|
||||
influx_clock_observe(&clock,"Wed, 11 Nov 2026 12:00:00 GMT",1000);
|
||||
assert(influx_clock_timestamp(&clock,500,&first));
|
||||
assert(influx_clock_timestamp(&clock,1500,&second) && second-first==1000);
|
||||
influx_batch_t b={0};assert(!influx_batch_due(&b,0));
|
||||
assert(influx_batch_add(&b,"air value=1i 12345",50));
|
||||
assert(!influx_batch_due(&b,10049) && influx_batch_due(&b,10050));
|
||||
assert(b.first_timestamp_ms==12345 && b.last_timestamp_ms==12345);
|
||||
for(int i=1;i<10;++i)assert(influx_batch_add(&b,"air value=2i 12346",100));
|
||||
assert(influx_batch_due(&b,100) && !influx_batch_add(&b,"air value=3i 12347",100));
|
||||
}
|
||||
int main(int argc,char **argv)
|
||||
{
|
||||
assert(argc==2);scenario=argv[1];
|
||||
if(!strcmp(scenario,"format"))test_format();
|
||||
else if(!strcmp(scenario,"clock"))test_clock();
|
||||
else {
|
||||
if(!strcmp(scenario,"retry"))failures_left=1;
|
||||
if(!strcmp(scenario,"overflow")) { failures_left=5;target=70; }
|
||||
if(!strcmp(scenario,"partial"))target=3;
|
||||
if(!strcmp(scenario,"max-body"))target=10;
|
||||
assert(net_influx_start(NULL)==ESP_OK);assert(task_entry);task_entry(NULL);
|
||||
assert(!s_task && !s_q && !s_http && clients==cleanups);
|
||||
assert(accepted+rejected+atomic_load(&s_dropped)==target && max_queue<=32);
|
||||
if(!strcmp(scenario,"normal"))assert(posts==3 && accepted==30 && clients==1 && gets==1 && successes==3);
|
||||
if(!strcmp(scenario,"retry"))assert(posts==4 && clients==2 && accepted==30);
|
||||
if(!strcmp(scenario,"overflow"))assert(atomic_load(&s_dropped)>0 && max_queue==32 && accepted>=32);
|
||||
if(!strcmp(scenario,"partial"))assert(posts==1 && samples_per_post[0]==3 && now_ms==11000);
|
||||
if(!strcmp(scenario,"max-body"))assert(posts==2 && accepted==10);
|
||||
if(!strcmp(scenario,"reject"))assert(posts==3 && accepted==20 && rejected==10);
|
||||
if(no_date())assert(posts==target && clients<=2);
|
||||
assert(status_ok);
|
||||
}
|
||||
printf("PASS: Influx %s\n",scenario);
|
||||
}
|
||||
Reference in New Issue
Block a user