log_csv.c File Reference

Reading a CSV log without knowing, in advance, what wrote it. More...

#include "log_csv.h"
#include <ctype.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "log_csv_int.h"
#include "log_fields.h"

Data Structures
struct	row_buf_t
	One row read from the file: its text, its fields and their counts. More...

Macros
#define	TAIL_CHUNK   256u
#define	TAIL_LIMIT   4096u

Functions
void	log_set_allocator (void *(*alloc_fn)(size_t), void(*free_fn)(void *))
	Where the sample arrays come from.
const char *	log_err_str (log_err_t err)
	Text for an import error.
static char *	trim (char *s)
	Strip white space from both ends of a string, in place.
static void	copy_str (char *dst, size_t dst_size, const char *src)
	Copy a string into a buffer, cut to fit without splitting a UTF-8 sequence.
static bool	ci_equal (const char *a, const char *b)
	Whether two strings are equal without regard to case.
static bool	ci_contains (const char *hay, const char *needle)
	Whether a string contains another, without regard to case.
void	log_csv_opts_default (log_csv_opts_t *opts)
	Set the default import options.
static bool	next_row (log_reader_t *r, char delimiter, row_buf_t *row)
	Read the next row that is not a blank line. false at end of input.
static bool	row_has_content (row_buf_t *row)
	Does this row have anything in it at all?
static void	guess_time_unit (const log_column_t *col, char *out, size_t out_size)
	Guess the unit of a time column.
static log_err_t	sniff_delimiter (log_source_t *src, const log_csv_opts_t *opts, char *out)
	Sniffing the delimiter, which has to happen before anything can be split into fields.
static int	tail_cut (log_source_t *src)
	Whether the source ends inside a row, read from its last bytes.
static log_err_t	scan_geometry (log_source_t *src, char delimiter, const log_csv_opts_t *opts, int max_rows, log_analysis_t *out)
	Pass one: how many columns there are, how many rows, and what the file looks like as evidence for a decimal convention.
static log_err_t	scan_columns (log_source_t *src, char delimiter, log_analysis_t *out)
	Pass two: what is in each column, now that the convention is settled.
static void	pick_time_column (const log_csv_opts_t *opts, log_analysis_t *out)
	Which column is the time axis.
log_err_t	log_csv_analyse (log_source_t *src, const log_csv_opts_t *opts, log_analysis_t *out)
	Two streaming passes: geometry and evidence, then per-column statistics.
void	log_data_free (log_data_t *data)
	Free the sample arrays of a build.
static int	cmp_float (const void *a, const void *b)
	Compare two floats for qsort().
static int	pick_columns (const log_analysis_t *a, const int *columns, int n_columns, int *pick)
	Which columns the plot gets: the ones asked for, or every numeric one that is not the time axis.
static void	name_fields (const log_analysis_t *a, const int *pick, int n_pick, const double *vmin, const double *vmax, log_data_t *out)
	What each series is called, what it is measured in, and what it is grouped with.
static void	undo_a_backwards_time_axis (const log_analysis_t *a, log_data_t *out, int count)
	A time axis that runs backwards means the wrong column was picked, so the samples are numbered instead and the axis loses its name – a plot against sample number is honest, and one against a column that is not time is not.
static log_err_t	read_series (log_source_t *src, const log_analysis_t *a, const int *pick, int n_pick, log_data_t *out, int *count_out)
	The time axis and the series, read together.
static void	series_stats (log_data_t *out, int count)
	What the samples add up to: how long, how fast, and the widest gap.
log_err_t	log_csv_build (log_source_t *src, const log_analysis_t *a, const int *columns, int n_columns, log_data_t *out)
	Materialise the selected columns.

Variables
static void *(*)	s_alloc (size_t) = malloc
static void(*)	s_free (void *) = free
static row_buf_t	s_row
static log_unit_tally_t	s_tally [LOG_MAX_COLUMNS]
static const char *const	k_time_names []

Detailed Description

Reading a CSV log without knowing, in advance, what wrote it.

A bench log can come from a transmitter, a charger, a datalogger or a spreadsheet, and they disagree about nearly everything: comma, semicolon or tab between fields, comma or dot for the decimal point, CRLF or LF between rows, a byte-order mark or not, quoted fields with doubled quotes inside them. So nothing here is assumed: the delimiter and the decimal convention are found by reading the file (see log_numbers.c for the evidence they are found from), and the row splitter handles the quoting and line-ending cases rather than trusting the file to be simple. The sources, the row splitter and the delimiter sniffer are in log_reader.c; the analysis and the build of the plotted arrays are here.

The source is behind read/rewind function pointers so the same reader runs against a file on the card and against a string in a test, and the allocator is injectable for the same reason: the parser can be held to an exact allocation count without a heap under it.

SPDX-License-Identifier: MIT

Function Documentation

ci_contains()

bool ci_contains ( const char * hay, const char * needle )

static

Whether a string contains another, without regard to case.

Parameters

hay	The string searched.
needle	The string to find.

Returns

true when needle occurs in hay, without regard to case.

ci_equal()

bool ci_equal ( const char * a, const char * b )

static

Whether two strings are equal without regard to case.

Parameters

a	The first string.
b	The second string.

Returns

true when the strings are equal without regard to case.

cmp_float()

int cmp_float ( const void * a, const void * b )

static

Compare two floats for qsort().

Parameters

a	Pointer to a float.
b	Pointer to a float.

Returns

negative, zero or positive as *a is less than, equal to or greater than *b.

copy_str()

void copy_str ( char * dst, size_t dst_size, const char * src )

static

Copy a string into a buffer, cut to fit without splitting a UTF-8 sequence.

Parameters

dst	Receives the copy, NUL-terminated.
dst_size	Size of dst, bytes.
src	The string to copy.

guess_time_unit()

void guess_time_unit ( const log_column_t * col, char * out, size_t out_size )

static

Guess the unit of a time column.

Parameters

col	The time column.
out	Receives the unit.
out_size	Size of out, bytes.

log_csv_analyse()

log_err_t log_csv_analyse	(	log_source_t *	src,
		const log_csv_opts_t *	opts,
		log_analysis_t *	out )

Two streaming passes: geometry and evidence, then per-column statistics.

Parameters

src	The input.
opts	The import options.
out	Receives the analysis.

Returns

LOG_OK, or the reason the analysis failed.

log_csv_build()

log_err_t log_csv_build	(	log_source_t *	src,
		const log_analysis_t *	a,
		const int *	columns,
		int	n_columns,
		log_data_t *	out )

Materialise the selected columns.

Parameters

src	The input.
a	The analysis.
columns	column indices to plot, or NULL to take the first numeric ones that are not the time axis
n_columns	Number of entries in columns.
out	Receives the data.

Returns

LOG_OK, or the reason the data was not built.

log_csv_opts_default()

void log_csv_opts_default

(

log_csv_opts_t *

opts

)

Set the default import options.

Parameters

opts	Receives the defaults.

log_data_free()

void log_data_free

(

log_data_t *

data

)

Free the sample arrays of a build.

Parameters

data	The data to free.

log_err_str()

const char * log_err_str

(

log_err_t

err

)

Text for an import error.

Parameters

err	The error.

Returns

A static text.

log_set_allocator()

void log_set_allocator	(	void *	alloc_fn (*)(size_t),
		void(*	free_fn )(void *) )

Where the sample arrays come from.

The firmware points this at PSRAM; the default is malloc/free, which is what the host tests want.

Parameters

alloc_fn	Allocates a block of bytes.
free_fn	Frees a block from alloc_fn.

name_fields()

void name_fields ( const log_analysis_t * a, const int * pick, int n_pick, const double * vmin, const double * vmax, log_data_t * out )

static

What each series is called, what it is measured in, and what it is grouped with.

The blackbox grouping when the names match, and otherwise by unit, which keeps related channels together on a screen that has to draw them somewhere.

Parameters

a	The analysis.
pick	The chosen column indices.
n_pick	Number of entries in pick.
vmin	Smallest value of each chosen column.
vmax	Largest value of each chosen column.
out	Receives the fields.

next_row()

bool next_row ( log_reader_t * r, char delimiter, row_buf_t * row )

static

Read the next row that is not a blank line. false at end of input.

Parameters

r	The row splitter.
delimiter	The field delimiter.
row	Receives the fields of the row.

Returns

true when a row was read; false at the end of input.

pick_columns()

int pick_columns ( const log_analysis_t * a, const int * columns, int n_columns, int * pick )

static

Which columns the plot gets: the ones asked for, or every numeric one that is not the time axis.

A column that is not numeric is dropped rather than refused, because a file with a text column beside its numbers is an ordinary file and not a broken one.

Parameters

a	The analysis.
columns	Column indices asked for, or NULL for every numeric column.
n_columns	Number of entries in columns.
pick	Receives the chosen column indices.

Returns

Number of columns chosen, at most LOG_MAX_SERIES.

pick_time_column()

void pick_time_column ( const log_csv_opts_t * opts, log_analysis_t * out )

static

Which column is the time axis.

Named first, then headed like one, then a first column that only ever increases – in that order, because a column called "time" is a statement and a column that happens to rise is a guess.

Parameters

opts	The import options.
out	The analysis; receives the time column and unit.

read_series()

log_err_t read_series ( log_source_t * src, const log_analysis_t * a, const int * pick, int n_pick, log_data_t * out, int * count_out )

static

The time axis and the series, read together.

One pass: a row's time and its values are the same row, and reading them separately would need the file twice or the rows held.

Parameters

src	The source.
a	The analysis.
pick	The chosen column indices.
n_pick	Number of entries in pick.
out	Receives the samples.
count_out	Receives the number of samples read.

Returns

LOG_OK, or the reason the samples could not be read.

row_has_content()

bool row_has_content ( row_buf_t * row )

static

Does this row have anything in it at all?

Parameters

row

The row.

Returns

true when a field is not white space.

scan_columns()

log_err_t scan_columns ( log_source_t * src, char delimiter, log_analysis_t * out )

static

Pass two: what is in each column, now that the convention is settled.

Bounded by the row count pass one arrived at, so a file that grew between the two passes is read to the length that was measured rather than to a different one.

Parameters

src	The source.
delimiter	The field delimiter.
out	The analysis from the first pass; receives the columns.

Returns

LOG_OK, or the reason the source could not be read.

scan_geometry()

log_err_t scan_geometry ( log_source_t * src, char delimiter, const log_csv_opts_t * opts, int max_rows, log_analysis_t * out )

static

Pass one: how many columns there are, how many rows, and what the file looks like as evidence for a decimal convention.

Two passes rather than one because the convention decides how a cell parses, and it is not known until every cell has voted. The alternative is holding the file, and the file is the one thing that does not fit.

Parameters

src	The source.
delimiter	The field delimiter.
opts	The import options.
max_rows	Most rows to count.
out	Receives the geometry and the evidence.

Returns

LOG_OK, or the reason the source could not be analysed.

series_stats()

void series_stats ( log_data_t * out, int count )

static

What the samples add up to: how long, how fast, and the widest gap.

The median interval rather than the mean, because one long gap in a log that is otherwise steady should not move the rate the screen reports. When there is no room to sort for a median, the mean is reported and the screen says which it is showing.

Parameters

out	The data; receives the statistics.
count	Number of samples in the time array.

sniff_delimiter()

log_err_t sniff_delimiter ( log_source_t * src, const log_csv_opts_t * opts, char * out )

static

Sniffing the delimiter, which has to happen before anything can be split into fields.

A source that cannot be rewound is a read error rather than a guess: every pass below starts from the top.

Parameters

src	The source; rewound and read again.
opts	The import options; a set delimiter is kept.
out	Receives the delimiter.

Returns

LOG_OK, or LOG_ERR_READ when the source cannot be rewound.

tail_cut()

int tail_cut ( log_source_t * src )

static

Whether the source ends inside a row, read from its last bytes.

Parameters

src	The source.

Returns

1 when the source ends inside a row, 0 when it does not, -1 when the last bytes cannot be read.

trim()

char * trim ( char * s )

static

Strip white space from both ends of a string, in place.

Parameters

s	The string, changed in place.

Returns

The first character after the leading white space.

undo_a_backwards_time_axis()

void undo_a_backwards_time_axis ( const log_analysis_t * a, log_data_t * out, int count )

static

A time axis that runs backwards means the wrong column was picked, so the samples are numbered instead and the axis loses its name – a plot against sample number is honest, and one against a column that is not time is not.

Parameters

a	The analysis.
out	The data; its time axis is replaced when it runs backwards.
count	Number of samples in the time array.

Variable Documentation

k_time_names

const char* const k_time_names[]

static

Initial value:

                                     = {
    "time", "timestamp", "zeit", "zeitstempel", "t", "elapsed", "clock",
    "sekunden", "seconds", "millis", "micros",
}