From a1944f0d70fed7b8ae772d1ec5038b4202a97458 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 13 Oct 2026 01:39:54 +0000 Subject: [PATCH 1/8] fix(asap-tools): send paced feeder ticks concurrently, with samples in (t, t+1s] Remote write sent one sub-second slot at a time, so at most one request was in flight per slot. At 200 samples/s per series ASAPQuery's ingest took about 45 ms per 1e4-row request and the feeder reached only 0.2-0.27M rows/s. Each tick now goes out concurrently. Its samples sit in (t, t+1s], the same left-open interval as ASAPQuery's one-second windows, so the reordering within a tick never crosses a window boundary and stays within the allowed lateness. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AnJ7LocWxguh3q4QWkCYae --- .../data-sources/paced-feeder/README.md | 13 ++-- .../data-sources/paced-feeder/src/main.rs | 69 +++++++++---------- .../data-sources/paced-feeder/src/workload.rs | 9 +-- 3 files changed, 46 insertions(+), 45 deletions(-) diff --git a/asap-tools/data-sources/paced-feeder/README.md b/asap-tools/data-sources/paced-feeder/README.md index 55a0ed82..0ff1d954 100644 --- a/asap-tools/data-sources/paced-feeder/README.md +++ b/asap-tools/data-sources/paced-feeder/README.md @@ -54,11 +54,14 @@ credentials. ## Pacing and stats Each tick's rows are split into `--batch-rows` requests and sent with up to -`--concurrency` in flight. For remote write, a tick sends its -`--samples-per-sec` sub-second slots in turn, and the next slot starts only -after every request of the current one returns, so each aggregation group's -samples reach ASAPQuery in timestamp order. ClickHouse sorts on insert, so it -gets the whole tick at once. +`--concurrency` in flight. The next tick starts only after every request of +the current one returns, so samples arrive out of order only within a tick. +A tick starting at `t` holds samples spaced evenly over `(t, t + 1 s]`, the +same left-open interval as ASAPQuery's one-second windows, so a tick never +spans two windows. Out-of-order samples within it are then safe for +ASAPQuery as long as its allowed lateness is at least 1 s (the default is +5 s). Prometheus' remote-write receiver needs an out-of-order window of at +least 1 s (`storage.tsdb.out_of_order_time_window`). `--samples-per-sec` is at most 1000, so timestamps (in ms) never repeat. The default batch size is 50,000 rows for remote write, which keeps a compressed request under axum's 2 MB body limit in ASAPQuery's ingest, and 100,000 for diff --git a/asap-tools/data-sources/paced-feeder/src/main.rs b/asap-tools/data-sources/paced-feeder/src/main.rs index 42db84de..8d1ae688 100644 --- a/asap-tools/data-sources/paced-feeder/src/main.rs +++ b/asap-tools/data-sources/paced-feeder/src/main.rs @@ -263,43 +263,40 @@ async fn main() -> Result<()> { let started = Instant::now(); let (mut rows, mut bytes) = (0, 0); - // ASAPQuery can drop samples behind a group's watermark, so remote write - // sends one slot at a time: a slot's requests all return before the - // next slot's are sent. ClickHouse sorts on insert, so it sends the - // whole tick at once. - let slots = (0..args.samples_per_sec).map(|slot| workload.slot_rows(tick, slot, start_ms)); - let waves: Vec> = match args.sink { - SinkKind::RemoteWrite => slots.collect(), - SinkKind::Clickhouse => vec![slots.flatten().collect()], - }; - for wave in waves { - let wave_rows = Arc::new(wave); - let mut tasks = tokio::task::JoinSet::new(); - for begin in (0..wave_rows.len()).step_by(batch_rows) { - let end = (begin + batch_rows).min(wave_rows.len()); - let (wave_rows, sink) = (Arc::clone(&wave_rows), Arc::clone(&sink)); - let permit = Arc::clone(&permits).acquire_owned().await?; - tasks.spawn(async move { - let _permit = permit; - let body = sink.encoder.encode(&wave_rows[begin..end]); - let bytes = body.len(); - sink.send(body).await.map(|()| (end - begin, bytes)) - }); - } + // A tick's requests go out concurrently, so a group's samples within + // the tick may arrive out of order. That is safe for ASAPQuery: the + // tick lies in one window (see `slot_rows`) and is under one second + // behind the watermark, within the allowed lateness. + let tick_rows: Arc> = Arc::new( + (0..args.samples_per_sec) + .flat_map(|slot| workload.slot_rows(tick, slot, start_ms)) + .collect(), + ); + let mut tasks = tokio::task::JoinSet::new(); + for begin in (0..tick_rows.len()).step_by(batch_rows) { + let end = (begin + batch_rows).min(tick_rows.len()); + let (tick_rows, sink) = (Arc::clone(&tick_rows), Arc::clone(&sink)); + let permit = Arc::clone(&permits).acquire_owned().await?; + tasks.spawn(async move { + let _permit = permit; + let body = sink.encoder.encode(&tick_rows[begin..end]); + let bytes = body.len(); + sink.send(body).await.map(|()| (end - begin, bytes)) + }); + } - while let Some(joined) = tasks.join_next().await { - // A panicked task counts as a failed request, so the stats of - // the ticks before it are still written. - match joined.map_err(anyhow::Error::from).and_then(|sent| sent) { - Ok((n_rows, n_bytes)) => { - rows += n_rows; - bytes += n_bytes; - } - Err(e) => { - failed_requests += 1; - if errors.len() < 10 { - errors.push(format!("tick {tick}: {e:#}")); - } + while let Some(joined) = tasks.join_next().await { + // A panicked task counts as a failed request, so the stats of + // the ticks before it are still written. + match joined.map_err(anyhow::Error::from).and_then(|sent| sent) { + Ok((n_rows, n_bytes)) => { + rows += n_rows; + bytes += n_bytes; + } + Err(e) => { + failed_requests += 1; + if errors.len() < 10 { + errors.push(format!("tick {tick}: {e:#}")); } } } diff --git a/asap-tools/data-sources/paced-feeder/src/workload.rs b/asap-tools/data-sources/paced-feeder/src/workload.rs index 5e68ea90..3b93561e 100644 --- a/asap-tools/data-sources/paced-feeder/src/workload.rs +++ b/asap-tools/data-sources/paced-feeder/src/workload.rs @@ -41,12 +41,13 @@ impl Workload { } /// Samples for sub-second slot `slot` of one-second tick `tick`, one per - /// series, with event time starting at `start_ms`. Slots within the second - /// are spaced evenly. + /// series. Slots are spaced evenly over `(t, t + 1 s]`, where `t` is the + /// tick's start: the same left-open interval as ASAPQuery's one-second + /// windows, so a tick never spans two of them. pub fn slot_rows(&self, tick: u64, slot: u32, start_ms: i64) -> Vec { let per_sec = u64::from(self.samples_per_sec); let slot = u64::from(slot); - let timestamp_ms = start_ms + tick as i64 * 1_000 + (slot * 1_000 / per_sec) as i64; + let timestamp_ms = start_ms + tick as i64 * 1_000 + ((slot + 1) * 1_000 / per_sec) as i64; let sample_index = tick * per_sec + slot; (0..self.num_series()) .map(|series| Row { @@ -98,7 +99,7 @@ mod tests { #[test] fn each_slot_has_every_series_at_its_timestamp() { let w = workload(); - for (slot, timestamp_ms) in [(0, 15_000), (1, 15_500)] { + for (slot, timestamp_ms) in [(0, 15_500), (1, 16_000)] { let rows = w.slot_rows(5, slot, 10_000); assert_eq!(rows.len(), 6); assert!(rows.iter().all(|r| r.timestamp_ms == timestamp_ms)); From a2196250b526ebe24fe3b52f69a24db6c5501fcc Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 13 Oct 2026 01:10:52 +0000 Subject: [PATCH 2/8] feat(asap-tools): generate the paced feeder's data from a workload table `--workload-table FILE` reads `workloads[0].e2e.metrics` of a sketch-bench workload table and sends its gauge and counter metrics (or only `--metric`): labels with `child_of` fan out under their parent, the others form a cross product; series get Zipf scales from a seeded rank shuffle; gauges are scaled Pareto draws and counters grow by their scale per sample. `--verify-facts` checks, for every label subset, the distinct label-value tuples of the generated series against the description. The `--groups` workload sends the same bytes as before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AnJ7LocWxguh3q4QWkCYae --- .../data-sources/paced-feeder/README.md | 95 +++- .../data-sources/paced-feeder/src/encode.rs | 204 +++++-- .../data-sources/paced-feeder/src/main.rs | 290 ++++++---- .../data-sources/paced-feeder/src/table.rs | 499 ++++++++++++++++++ .../data-sources/paced-feeder/src/workload.rs | 221 +++++--- 5 files changed, 1087 insertions(+), 222 deletions(-) create mode 100644 asap-tools/data-sources/paced-feeder/src/table.rs diff --git a/asap-tools/data-sources/paced-feeder/README.md b/asap-tools/data-sources/paced-feeder/README.md index 0ff1d954..c7362364 100644 --- a/asap-tools/data-sources/paced-feeder/README.md +++ b/asap-tools/data-sources/paced-feeder/README.md @@ -8,17 +8,79 @@ one second of event time per wall-clock second, to either: It drives the data-plane vs ClickHouse evaluation (`asap-tools/experiments/DATA_PLANE_CLICKHOUSE_EVAL_PLAN.md`). Two feeders -started with the same workload flags send identical data, so each system gets +started with the same workload flags (and table) send identical data, so each system gets the same stream without a dataset file on disk. ## Workload +The feeder generates either the fixed `--groups` workload or the metrics of a +workload table (`--workload-table`). Either way, a value depends only on +`(--seed, metric, series, sample index)`, never on wall-clock time or the sink, +and each series emits `--samples-per-sec` samples per second, evenly spaced. + +### `--groups` workload + - `--groups C` values of `label_0` (`g0`, `g1`, …) and `--series-per-group s` - values of `instance` (`i0`, `i1`, …) per group: `C·s` series. -- Each series emits `--samples-per-sec` samples per second, evenly spaced. + values of `instance` (`i0`, `i1`, …) per group: `C·s` series of one gauge, + named by `--metric` (default `data`). - Values are Pareto(`--pareto-shape`, `--pareto-scale`), all ≥ the scale. -- A value depends only on `(--seed, series, sample index)`, never on wall-clock - time or the sink. +- `--samples-per-sec` defaults to 1. + +### `--workload-table` workload + +`--workload-table FILE` reads `workloads[0].e2e.metrics` of a sketch-bench +workload table (`scripts/export_autosketch_eval_table.py`) and sends every +metric in it, or only `--metric NAME`. For example: + +```json +"data": {"type": "gauge", + "labels": [{"name": "job", "cardinality": 10}, + {"name": "label_0", "cardinality": 1000, "child_of": "job"}], + "samples_per_sec_per_series": 200, + "values": {"value": {"zipf": 1.1, "keys": 10000}, "dist": {"pareto": 2.0}}}, +"data_total": {"type": "counter", "labels": […], "samples_per_sec_per_series": 200, + "values": {"increment": {"zipf": 1.1, "keys": 10000}}} +``` + +- **Labels.** A label has `cardinality` values, or, with `child_of`, that many + under each value of its parent, which must be listed earlier. A root label's + values are `0`, `1`, …; a child's are its parent's value, a `.`, and the + child index (`job="3"`, `label_0="3.17"`), so a child value names its parent + and is unique across parents. Labels without `child_of` form a cross + product. The metric has one series per combination: the product of all + cardinalities (1e4 above). +- **Key skew.** `zipf` θ over `keys` keys, where a key is a series (`keys` + must equal the number of series). The series of Zipf rank `r` (0 the + heaviest) has scale `N·(r+1)^-θ / Σ_j (j+1)^-θ`, so scales average 1. Ranks + are a shuffle seeded by `--seed` alone, so the heavy series spread across + parents (not all under `job="0"`) and a series is equally heavy in every + metric of the table. +- **Gauge** (`value` and `dist`): sample `n` of series `i` is + `scale_i · U^(-1/a)` with `U` a hashed uniform in (0, 1]: Pareto(a, 1) + scaled by the series' weight, which keeps the tail index `a`. +- **Counter** (`increment`): sample `n` of series `i` is `(n+1) · scale_i`, + i.e. a constant increment `scale_i` per sample, starting at the first + sample and never resetting, so `rate()` is `scale_i · samples/s`. +- `--samples-per-sec` overrides `samples_per_sec_per_series`, e.g. to scale a + run down. Without it, the sent metrics must share one rate. +- Other metric types (the multi-grouping `records` metrics) are rejected. + +`--verify-facts` checks the generated series against the description without +sending anything: for every non-empty subset of a metric's labels it prints +the expected number of distinct label-value tuples (the product of the +cardinalities of the subset plus every label's `child_of` ancestors) and the +number among the generated series, and exits non-zero if any differ. + +```sh +$ paced_feeder --workload-table synthetic-templatesclassic-shared1.json --verify-facts +data: 10000 series + labels expected generated + job 10 10 + label_0 10000 10000 + job,label_0 10000 10000 +data_total: 10000 series + … +``` Event time starts at the next whole wall-clock second (or `--start-ms`), so `NOW()`-relative queries see the data as it arrives. @@ -39,15 +101,36 @@ target/release/paced_feeder --sink clickhouse \ --url http://localhost:8123 --table data \ --groups 1000 --series-per-group 100 --duration-secs 900 \ --stats-out clickhouse-feed.json + +# A workload table's metrics (data and data_total) to ASAPQuery +target/release/paced_feeder --sink remote-write \ + --url http://localhost:9090/api/v1/write \ + --workload-table synthetic-templatesclassic-shared1.json --duration-secs 900 ``` -The ClickHouse table must have these columns (any engine and ordering): +Remote write sends each metric's name as `__name__` and its labels sorted by +name, as Prometheus requires; it also works against Prometheus' +remote-write receiver (`--web.enable-remote-write-receiver`). + +With `--groups`, the ClickHouse table (`--table`) must have these columns (any +engine and ordering): ```sql CREATE TABLE data (ts DateTime64(3), label_0 String, instance String, value Float64) ENGINE = MergeTree ORDER BY (label_0, instance, ts); ``` +With `--workload-table`, each metric goes to the table named after it, with +columns `ts`, then the metric's labels in the description's order, then +`value`. For the example above: + +```sql +CREATE TABLE data (ts DateTime64(3), job String, label_0 String, value Float64) +ENGINE = MergeTree ORDER BY (job, label_0, ts); +CREATE TABLE data_total (ts DateTime64(3), job String, label_0 String, value Float64) +ENGINE = MergeTree ORDER BY (job, label_0, ts); +``` + `CLICKHOUSE_USER` / `CLICKHOUSE_PASSWORD` (or the matching flags) set credentials. diff --git a/asap-tools/data-sources/paced-feeder/src/encode.rs b/asap-tools/data-sources/paced-feeder/src/encode.rs index 32a4caba..bb6249b2 100644 --- a/asap-tools/data-sources/paced-feeder/src/encode.rs +++ b/asap-tools/data-sources/paced-feeder/src/encode.rs @@ -1,7 +1,7 @@ //! Wire encoders for the two sinks. Each series' labels are encoded once up //! front, so the per-sample cost at high rates is just the value and timestamp. -use crate::workload::{Row, Workload}; +use crate::workload::{Metric, Row}; use prost::encoding::{encode_key, encode_varint, encoded_len_varint, WireType}; pub trait Encoder: Send + Sync { @@ -12,21 +12,27 @@ pub trait Encoder: Send + Sync { /// `TimeSeries` (labels + one sample) per row. pub struct RemoteWriteEncoder { /// Encoded `Label` fields (`TimeSeries` field 1) for each series, sorted by - /// label name as Prometheus requires: `__name__`, `instance`, `label_0`. + /// label name as Prometheus requires, `__name__` included. series_labels: Vec>, } impl RemoteWriteEncoder { - pub fn new(workload: &Workload, metric: &str) -> Self { - let series_labels = (0..workload.num_series()) - .map(|series| { - let (label_0, instance) = workload.series_labels(series); + /// Series of `metric`, sent with `__name__` = `name`. + pub fn new(metric: &Metric, name: &str) -> Self { + let series_labels = metric + .series + .iter() + .map(|values| { + let mut labels: Vec<(&str, &str)> = metric + .labels + .iter() + .map(|l| l.name.as_str()) + .zip(values.iter().map(String::as_str)) + .collect(); + labels.push(("__name__", name)); + labels.sort_unstable(); let mut buf = Vec::new(); - for (name, value) in [ - ("__name__", metric), - ("instance", instance.as_str()), - ("label_0", label_0.as_str()), - ] { + for (name, value) in labels { let mut label = Vec::new(); encode_bytes_field(1, name.as_bytes(), &mut label); encode_bytes_field(2, value.as_bytes(), &mut label); @@ -63,23 +69,31 @@ impl Encoder for RemoteWriteEncoder { } } -/// ClickHouse `RowBinary` for columns `(ts DateTime64(3), label_0 String, -/// instance String, value Float64)`. +/// ClickHouse `RowBinary` for columns `ts DateTime64(3)`, then each label of +/// the metric as `String` in its order, then `value Float64`. pub struct RowBinaryEncoder { - /// `label_0` then `instance`, each as a varint length plus bytes. + /// Each series' label values, each as a varint length plus bytes. series_labels: Vec>, } -/// The column list `RowBinaryEncoder` writes, in order. -pub const ROW_BINARY_COLUMNS: &str = "ts, label_0, instance, value"; +/// The column list `RowBinaryEncoder` writes for `metric`, in order. +pub fn row_binary_columns(metric: &Metric) -> String { + let labels = metric.labels.iter().map(|l| l.name.as_str()); + std::iter::once("ts") + .chain(labels) + .chain(std::iter::once("value")) + .collect::>() + .join(", ") +} impl RowBinaryEncoder { - pub fn new(workload: &Workload) -> Self { - let series_labels = (0..workload.num_series()) - .map(|series| { - let (label_0, instance) = workload.series_labels(series); + pub fn new(metric: &Metric) -> Self { + let series_labels = metric + .series + .iter() + .map(|values| { let mut buf = Vec::new(); - for s in [label_0, instance] { + for s in values { encode_varint(s.len() as u64, &mut buf); buf.extend_from_slice(s.as_bytes()); } @@ -112,6 +126,8 @@ fn encode_bytes_field(tag: u32, bytes: &[u8], buf: &mut Vec) { #[cfg(test)] mod tests { use super::*; + use crate::table::series_labels; + use crate::workload::{Kind, LabelSpec, Metric, Workload}; use prost::Message; // Mirrors the remote write types in @@ -143,28 +159,52 @@ mod tests { timestamp: i64, } - fn workload() -> Workload { + fn grouped() -> Workload { + Workload::grouped("data", 1, 2, 3, 1, 1.5, 1.0) + } + + /// Labels out of name order, an uppercase one sorting before `__name__`. + fn multi_label() -> Workload { + let label = |name: &str, cardinality, child_of: Option<&str>| LabelSpec { + name: name.to_string(), + cardinality, + child_of: child_of.map(str::to_string), + }; + let labels = vec![ + label("zone", 2, None), + label("job", 2, None), + label("label_0", 3, Some("job")), + label("Az", 1, None), + ]; + let series = series_labels(&labels).unwrap(); + let metric = Metric { + name: "data_total".to_string(), + scales: vec![1.0; series.len()], + labels, + series, + kind: Kind::Counter, + seed: 0, + }; Workload { - seed: 1, - groups: 2, - series_per_group: 3, samples_per_sec: 1, - pareto_shape: 1.5, - pareto_scale: 1.0, + metrics: vec![metric], } } + fn decode_remote_write(body: &[u8]) -> WriteRequest { + let raw = snap::raw::Decoder::new().decompress_vec(body).unwrap(); + WriteRequest::decode(raw.as_slice()).unwrap() + } + #[test] fn remote_write_decodes_to_one_series_per_row() { - let w = workload(); - let rows = w.slot_rows(2, 0, 1_700_000_000_000); - let body = RemoteWriteEncoder::new(&w, "data").encode(&rows); - - let raw = snap::raw::Decoder::new().decompress_vec(&body).unwrap(); - let request = WriteRequest::decode(raw.as_slice()).unwrap(); + let w = grouped(); + let rows = w.slot_rows(0, 2, 0, 1_700_000_000_000); + let request = + decode_remote_write(&RemoteWriteEncoder::new(&w.metrics[0], "data").encode(&rows)); assert_eq!(request.timeseries.len(), rows.len()); for (ts, row) in request.timeseries.iter().zip(&rows) { - let (label_0, instance) = w.series_labels(row.series); + let values = &w.metrics[0].series[row.series]; let labels: Vec<(&str, &str)> = ts .labels .iter() @@ -174,8 +214,8 @@ mod tests { labels, vec![ ("__name__", "data"), - ("instance", instance.as_str()), - ("label_0", label_0.as_str()), + ("instance", values[1].as_str()), + ("label_0", values[0].as_str()), ] ); assert_eq!( @@ -189,29 +229,93 @@ mod tests { } #[test] - fn row_binary_round_trips_every_column() { - let w = workload(); - let rows = w.slot_rows(0, 0, 42_000); - let buf = RowBinaryEncoder::new(&w).encode(&rows); + fn remote_write_sorts_labels_by_name() { + let w = multi_label(); + let metric = &w.metrics[0]; + let rows = w.slot_rows(0, 0, 0, 1_000); + let request = + decode_remote_write(&RemoteWriteEncoder::new(metric, &metric.name).encode(&rows)); + assert_eq!(request.timeseries.len(), 12); + for (ts, row) in request.timeseries.iter().zip(&rows) { + let [zone, job, label_0, az] = metric.series[row.series].as_slice() else { + panic!("four labels"); + }; + let labels: Vec<(&str, &str)> = ts + .labels + .iter() + .map(|l| (l.name.as_str(), l.value.as_str())) + .collect(); + assert_eq!( + labels, + vec![ + ("Az", az.as_str()), + ("__name__", "data_total"), + ("job", job.as_str()), + ("label_0", label_0.as_str()), + ("zone", zone.as_str()), + ] + ); + } + } - let mut cursor = buf.as_slice(); + /// Decodes RowBinary rows of `ts`, `num_labels` strings and `value`. + fn decode_row_binary(buf: &[u8], num_labels: usize) -> Vec<(i64, Vec, f64)> { + let mut cursor = buf; let mut take = |n: usize| -> Vec { let (head, rest) = cursor.split_at(n); cursor = rest; head.to_vec() }; - for row in &rows { + let mut rows = Vec::new(); + let mut remaining = buf.len(); + while remaining > 0 { let ts = i64::from_le_bytes(take(8).try_into().unwrap()); // Labels are short, so their varint length is one byte. - let label_0_len = take(1)[0] as usize; - let label_0 = String::from_utf8(take(label_0_len)).unwrap(); - let instance_len = take(1)[0] as usize; - let instance = String::from_utf8(take(instance_len)).unwrap(); + let labels: Vec = (0..num_labels) + .map(|_| { + let len = take(1)[0] as usize; + String::from_utf8(take(len)).unwrap() + }) + .collect(); let value = f64::from_le_bytes(take(8).try_into().unwrap()); - assert_eq!(ts, row.timestamp_ms); - assert_eq!((label_0, instance), w.series_labels(row.series)); - assert_eq!(value, row.value); + remaining -= 16 + labels.iter().map(|l| 1 + l.len()).sum::(); + rows.push((ts, labels, value)); } - assert!(cursor.is_empty()); + rows + } + + #[test] + fn row_binary_round_trips_every_column() { + let w = grouped(); + let metric = &w.metrics[0]; + assert_eq!(row_binary_columns(metric), "ts, label_0, instance, value"); + let rows = w.slot_rows(0, 0, 0, 42_000); + let decoded = decode_row_binary(&RowBinaryEncoder::new(metric).encode(&rows), 2); + let expected: Vec<_> = rows + .iter() + .map(|r| (r.timestamp_ms, metric.series[r.series].clone(), r.value)) + .collect(); + assert_eq!(decoded, expected); + } + + #[test] + fn row_binary_writes_labels_in_description_order() { + let w = multi_label(); + let metric = &w.metrics[0]; + assert_eq!( + row_binary_columns(metric), + "ts, zone, job, label_0, Az, value" + ); + let rows = w.slot_rows(0, 3, 0, 0); + let decoded = decode_row_binary(&RowBinaryEncoder::new(metric).encode(&rows), 4); + assert_eq!(decoded.len(), 12); + assert_eq!( + decoded[4], + ( + 4_000, + vec!["0".into(), "1".into(), "1.1".into(), "0".into()], + 4.0 + ) + ); } } diff --git a/asap-tools/data-sources/paced-feeder/src/main.rs b/asap-tools/data-sources/paced-feeder/src/main.rs index 8d1ae688..f7e1efff 100644 --- a/asap-tools/data-sources/paced-feeder/src/main.rs +++ b/asap-tools/data-sources/paced-feeder/src/main.rs @@ -7,11 +7,12 @@ //! is written to `--stats-out` so a run can be checked for keeping up. mod encode; +mod table; mod workload; use anyhow::{bail, Context, Result}; use clap::{Parser, ValueEnum}; -use encode::{Encoder, RemoteWriteEncoder, RowBinaryEncoder, ROW_BINARY_COLUMNS}; +use encode::{row_binary_columns, Encoder, RemoteWriteEncoder, RowBinaryEncoder}; use serde::Serialize; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -30,42 +31,55 @@ enum SinkKind { #[derive(Debug, Parser)] #[command(about = "Send a seeded synthetic workload in real time to ASAPQuery or ClickHouse")] struct Args { - #[arg(long, value_enum)] - sink: SinkKind, + #[arg(long, value_enum, required_unless_present = "verify_facts")] + sink: Option, /// Remote write endpoint (`http://host:port/api/v1/write`) or ClickHouse /// HTTP base URL (`http://host:8123`). + #[arg(long, required_unless_present = "verify_facts")] + url: Option, + /// With `--groups`: the metric name sent as `__name__` (remote write + /// only; default `data`). With `--workload-table`: send only this metric. #[arg(long)] - url: String, - /// Metric name sent as `__name__` (remote write only). - #[arg(long, default_value = "data")] - metric: String, - /// Target table, optionally `database.table` (ClickHouse only), made of - /// `[A-Za-z0-9_]` characters. Must have columns `ts DateTime64(3), label_0 String, instance String, value Float64`. - #[arg(long, default_value = "data")] + metric: Option, + /// Target table, optionally `database.table` (ClickHouse with `--groups` + /// only), made of `[A-Za-z0-9_]` characters. Must have columns `ts DateTime64(3), label_0 String, instance String, value Float64`. + #[arg(long, default_value = "data", conflicts_with = "workload_table")] table: String, #[arg(long, env = "CLICKHOUSE_USER")] clickhouse_user: Option, #[arg(long, env = "CLICKHOUSE_PASSWORD")] clickhouse_password: Option, + /// Generate the metrics of a sketch-bench workload table's + /// `workloads[0].e2e.metrics` instead of the `--groups` workload. Each + /// metric's ClickHouse table is named after it. + #[arg(long, conflicts_with_all = ["groups", "series_per_group", "pareto_shape", "pareto_scale"])] + workload_table: Option, /// Number of `label_0` values (C). - #[arg(long)] - groups: usize, + #[arg(long, required_unless_present = "workload_table")] + groups: Option, /// Number of `instance` values per group (s). + #[arg(long, required_unless_present = "workload_table")] + series_per_group: Option, + /// Samples per second per series. Defaults to 1 with `--groups`, and to + /// the table's `samples_per_sec_per_series` with `--workload-table`. #[arg(long)] - series_per_group: usize, - #[arg(long, default_value_t = 1)] - samples_per_sec: u32, + samples_per_sec: Option, #[arg(long, default_value_t = 1.5)] pareto_shape: f64, #[arg(long, default_value_t = 1.0)] pareto_scale: f64, #[arg(long, default_value_t = 42)] seed: u64, + /// Print, for every subset of each metric's labels, the expected and the + /// generated number of distinct label-value tuples, then exit without + /// sending; non-zero if any differ. + #[arg(long)] + verify_facts: bool, /// Seconds of event time to send. - #[arg(long)] - duration_secs: u64, + #[arg(long, required_unless_present = "verify_facts")] + duration_secs: Option, /// Event time of the first tick, in ms since the epoch. Defaults to the /// next whole wall-clock second, so `NOW()`-relative queries see live data. #[arg(long)] @@ -98,11 +112,14 @@ struct TickStats { #[derive(Debug, Serialize)] struct RunStats { sink: SinkKind, - groups: usize, - series_per_group: usize, + /// Unset with `--workload-table`. + groups: Option, + series_per_group: Option, + pareto_shape: Option, + pareto_scale: Option, + workload_table: Option, + metrics: Vec, samples_per_sec: u32, - pareto_shape: f64, - pareto_scale: f64, seed: u64, start_ms: i64, duration_secs: u64, @@ -120,50 +137,67 @@ struct RunStats { ticks: Vec, } -struct Sink { - client: reqwest::Client, +/// Where one metric's rows go: the request to send them with and their encoder. +struct Target { request: reqwest::RequestBuilder, encoder: Box, } +struct Sink { + client: reqwest::Client, + /// One per metric of the workload, in order. + targets: Vec, +} + impl Sink { - fn new(args: &Args, workload: &Workload) -> Result { + fn new(args: &Args, sink: SinkKind, url: &str, workload: &Workload) -> Result { let client = reqwest::Client::builder() .pool_max_idle_per_host(args.concurrency) .build()?; - let (request, encoder): (_, Box) = match args.sink { - SinkKind::RemoteWrite => ( - client - .post(&args.url) - .header("Content-Encoding", "snappy") - .header("Content-Type", "application/x-protobuf") - .header("X-Prometheus-Remote-Write-Version", "0.1.0"), - Box::new(RemoteWriteEncoder::new(workload, &args.metric)), - ), - SinkKind::Clickhouse => { - let query = format!( - "INSERT INTO {} ({ROW_BINARY_COLUMNS}) FORMAT RowBinary", - args.table - ); - let mut request = client.post(&args.url).query(&[("query", query)]); - if let Some(user) = &args.clickhouse_user { - request = request.header("X-ClickHouse-User", user); - } - if let Some(password) = &args.clickhouse_password { - request = request.header("X-ClickHouse-Key", password); + let targets = workload + .metrics + .iter() + .map(|metric| match sink { + SinkKind::RemoteWrite => Target { + request: client + .post(url) + .header("Content-Encoding", "snappy") + .header("Content-Type", "application/x-protobuf") + .header("X-Prometheus-Remote-Write-Version", "0.1.0"), + encoder: Box::new(RemoteWriteEncoder::new(metric, &metric.name)), + }, + SinkKind::Clickhouse => { + let table = if args.workload_table.is_some() { + &metric.name + } else { + &args.table + }; + let query = format!( + "INSERT INTO {table} ({}) FORMAT RowBinary", + row_binary_columns(metric) + ); + let mut request = client.post(url).query(&[("query", query)]); + if let Some(user) = &args.clickhouse_user { + request = request.header("X-ClickHouse-User", user); + } + if let Some(password) = &args.clickhouse_password { + request = request.header("X-ClickHouse-Key", password); + } + Target { + request, + encoder: Box::new(RowBinaryEncoder::new(metric)), + } } - (request, Box::new(RowBinaryEncoder::new(workload))) - } - }; - Ok(Self { - client, - request, - encoder, - }) + }) + .collect(); + Ok(Self { client, targets }) } - async fn send(&self, body: Vec) -> Result<()> { - let request = self + async fn send(&self, metric: usize, rows: &[Row]) -> Result { + let target = &self.targets[metric]; + let body = target.encoder.encode(rows); + let bytes = body.len(); + let request = target .request .try_clone() .context("request builder must be clonable")? @@ -175,7 +209,7 @@ impl Sink { let text = response.text().await.unwrap_or_default(); bail!("HTTP {status}: {}", text.trim()); } - Ok(()) + Ok(bytes) } } @@ -199,17 +233,81 @@ async fn wait_for_wall_clock(epoch_ms: i64) -> Instant { Instant::now() } +fn build_workload(args: &Args) -> Result { + if let Some(path) = &args.workload_table { + return table::load( + path, + args.metric.as_deref(), + args.samples_per_sec, + args.seed, + ); + } + let (groups, series_per_group) = ( + args.groups.unwrap_or_default(), + args.series_per_group.unwrap_or_default(), + ); + if groups == 0 || series_per_group == 0 { + bail!("--groups and --series-per-group must be positive"); + } + if args.pareto_shape <= 0.0 || args.pareto_scale <= 0.0 { + bail!("--pareto-shape and --pareto-scale must be positive"); + } + Ok(Workload::grouped( + args.metric.as_deref().unwrap_or("data"), + args.seed, + groups, + series_per_group, + args.samples_per_sec.unwrap_or(1), + args.pareto_shape, + args.pareto_scale, + )) +} + +/// Prints the facts check of every metric; true if every fact holds. +fn verify_facts(workload: &Workload) -> bool { + let mut ok = true; + for metric in &workload.metrics { + println!("{}: {} series", metric.name, metric.series.len()); + println!(" {:<40} {:>12} {:>12}", "labels", "expected", "generated"); + for fact in table::facts(&metric.labels, &metric.series) { + let holds = fact.expected == fact.actual; + ok &= holds; + println!( + " {:<40} {:>12} {:>12}{}", + fact.labels.join(","), + fact.expected, + fact.actual, + if holds { "" } else { " MISMATCH" } + ); + } + } + ok +} + #[tokio::main] async fn main() -> Result<()> { let args = Args::parse(); - if args.groups == 0 || args.series_per_group == 0 || args.samples_per_sec == 0 { - bail!("--groups, --series-per-group and --samples-per-sec must be positive"); + let workload = build_workload(&args)?; + if workload.samples_per_sec == 0 { + bail!("samples per second must be positive"); } // Timestamps are in ms, so more samples per second would repeat a timestamp // within a series, which remote write drops and ClickHouse keeps. - if args.samples_per_sec > 1_000 { - bail!("--samples-per-sec must be at most 1000"); + if workload.samples_per_sec > 1_000 { + bail!("samples per second must be at most 1000"); + } + if args.verify_facts { + if !verify_facts(&workload) { + bail!("the generated series do not match the description"); + } + return Ok(()); } + let (Some(sink_kind), Some(url), Some(duration_secs)) = + (args.sink, args.url.as_deref(), args.duration_secs) + else { + bail!("--sink, --url and --duration-secs are required"); + }; + // The table name is interpolated into the INSERT query. if args.table.is_empty() || !args @@ -219,10 +317,7 @@ async fn main() -> Result<()> { { bail!("--table must be made of [A-Za-z0-9_.] characters"); } - if args.pareto_shape <= 0.0 || args.pareto_scale <= 0.0 { - bail!("--pareto-shape and --pareto-scale must be positive"); - } - let batch_rows = args.batch_rows.unwrap_or(match args.sink { + let batch_rows = args.batch_rows.unwrap_or(match sink_kind { SinkKind::RemoteWrite => 50_000, SinkKind::Clickhouse => 100_000, }); @@ -230,34 +325,25 @@ async fn main() -> Result<()> { bail!("--batch-rows and --concurrency must be positive"); } - let workload = Workload { - seed: args.seed, - groups: args.groups, - series_per_group: args.series_per_group, - samples_per_sec: args.samples_per_sec, - pareto_shape: args.pareto_shape, - pareto_scale: args.pareto_scale, - }; - let sink = Arc::new(Sink::new(&args, &workload)?); + let sink = Arc::new(Sink::new(&args, sink_kind, url, &workload)?); let permits = Arc::new(Semaphore::new(args.concurrency)); let start_ms = args.start_ms.unwrap_or_else(next_whole_second_ms); + let metric_names: Vec = workload.metrics.iter().map(|m| m.name.clone()).collect(); eprintln!( - "paced_feeder: {:?} -> {}, {} series x {}/s = {} rows/s for {}s, start_ms={start_ms}", - args.sink, - args.url, + "paced_feeder: {sink_kind:?} -> {url}, {} ({} series) x {}/s = {} rows/s for {duration_secs}s, start_ms={start_ms}", + metric_names.join(", "), workload.num_series(), - args.samples_per_sec, + workload.samples_per_sec, workload.rows_per_tick(), - args.duration_secs ); let base = wait_for_wall_clock(start_ms).await; - let mut ticks = Vec::with_capacity(args.duration_secs as usize); + let mut ticks = Vec::with_capacity(duration_secs as usize); let mut errors: Vec = Vec::new(); let mut failed_requests = 0; - for tick in 0..args.duration_secs { + for tick in 0..duration_secs { let scheduled = base + Duration::from_secs(tick); tokio::time::sleep_until(scheduled.into()).await; let started = Instant::now(); @@ -267,22 +353,23 @@ async fn main() -> Result<()> { // the tick may arrive out of order. That is safe for ASAPQuery: the // tick lies in one window (see `slot_rows`) and is under one second // behind the watermark, within the allowed lateness. - let tick_rows: Arc> = Arc::new( - (0..args.samples_per_sec) - .flat_map(|slot| workload.slot_rows(tick, slot, start_ms)) - .collect(), - ); let mut tasks = tokio::task::JoinSet::new(); - for begin in (0..tick_rows.len()).step_by(batch_rows) { - let end = (begin + batch_rows).min(tick_rows.len()); - let (tick_rows, sink) = (Arc::clone(&tick_rows), Arc::clone(&sink)); - let permit = Arc::clone(&permits).acquire_owned().await?; - tasks.spawn(async move { - let _permit = permit; - let body = sink.encoder.encode(&tick_rows[begin..end]); - let bytes = body.len(); - sink.send(body).await.map(|()| (end - begin, bytes)) - }); + for metric in 0..workload.metrics.len() { + let metric_rows: Arc> = Arc::new( + (0..workload.samples_per_sec) + .flat_map(|slot| workload.slot_rows(metric, tick, slot, start_ms)) + .collect(), + ); + for begin in (0..metric_rows.len()).step_by(batch_rows) { + let end = (begin + batch_rows).min(metric_rows.len()); + let (metric_rows, sink) = (Arc::clone(&metric_rows), Arc::clone(&sink)); + let permit = Arc::clone(&permits).acquire_owned().await?; + tasks.spawn(async move { + let _permit = permit; + let sent = sink.send(metric, &metric_rows[begin..end]).await; + sent.map(|bytes| (end - begin, bytes)) + }); + } } while let Some(joined) = tasks.join_next().await { @@ -326,18 +413,21 @@ async fn main() -> Result<()> { // A run that keeps up still spans the full duration, even though the last // tick's work ends before its second does. - let wall_secs = base.elapsed().as_secs_f64().max(args.duration_secs as f64); + let wall_secs = base.elapsed().as_secs_f64().max(duration_secs as f64); let rows_sent: usize = ticks.iter().map(|t| t.rows).sum(); + let grouped = args.workload_table.is_none(); let run = RunStats { - sink: args.sink, + sink: sink_kind, groups: args.groups, series_per_group: args.series_per_group, - samples_per_sec: args.samples_per_sec, - pareto_shape: args.pareto_shape, - pareto_scale: args.pareto_scale, + pareto_shape: grouped.then_some(args.pareto_shape), + pareto_scale: grouped.then_some(args.pareto_scale), + workload_table: args.workload_table.clone(), + metrics: metric_names, + samples_per_sec: workload.samples_per_sec, seed: args.seed, start_ms, - duration_secs: args.duration_secs, + duration_secs, target_rows_per_sec: workload.rows_per_tick(), rows_sent, bytes_sent: ticks.iter().map(|t| t.bytes).sum(), diff --git a/asap-tools/data-sources/paced-feeder/src/table.rs b/asap-tools/data-sources/paced-feeder/src/table.rs new file mode 100644 index 00000000..e6f73329 --- /dev/null +++ b/asap-tools/data-sources/paced-feeder/src/table.rs @@ -0,0 +1,499 @@ +//! Metrics from a sketch-bench workload table's `workloads[0].e2e.metrics`, +//! and the facts check that the generated series match their description. +//! +//! A label has `cardinality` values, or, with `child_of`, that many under each +//! value of its parent. A root label's values are `0`, `1`, …; a child's are +//! its parent's value, a `.`, and the child index (`job="3"`, +//! `label_0="3.17"`), so a child value names its parent and is unique across +//! parents. Labels without `child_of` form a cross product, and the metric +//! has the product of all cardinalities as series. + +use crate::workload::{splitmix64, Kind, LabelSpec, Metric, Workload}; +use anyhow::{bail, ensure, Context, Result}; +use serde::Deserialize; +use serde_json::Value; +use std::collections::{BTreeMap, HashSet}; + +#[derive(Deserialize)] +struct Table { + workloads: Vec, +} + +#[derive(Deserialize)] +struct TableWorkload { + e2e: E2e, +} + +#[derive(Deserialize)] +struct E2e { + metrics: BTreeMap, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct MetricSpec { + #[serde(rename = "type")] + kind: String, + labels: Vec, + samples_per_sec_per_series: u32, + values: Value, +} + +/// Zipf(`zipf`) skew over `keys` keys; a key is a series. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct KeySkew { + zipf: f64, + keys: u64, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct GaugeValues { + value: KeySkew, + dist: ParetoDist, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct ParetoDist { + pareto: f64, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct CounterValues { + increment: KeySkew, +} + +/// The metrics of `workloads[0].e2e.metrics` in the table at `path`, in name +/// order, or only `only`. `samples_per_sec` overrides every metric's rate. +pub fn load( + path: &str, + only: Option<&str>, + samples_per_sec: Option, + seed: u64, +) -> Result { + let text = std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?; + let table: Table = serde_json::from_str(&text) + .with_context(|| format!("{path}: no workloads[0].e2e.metrics"))?; + let Some(first) = table.workloads.into_iter().next() else { + bail!("{path}: `workloads` is empty"); + }; + let mut metrics = Vec::new(); + let mut rates = Vec::new(); + for (name, spec) in &first.e2e.metrics { + if only.is_some_and(|only| only != name) { + continue; + } + let (metric, rate) = + metric(name, spec, seed).with_context(|| format!("metric `{name}`"))?; + metrics.push(metric); + rates.push(rate); + } + if let Some(only) = only { + ensure!(!metrics.is_empty(), "{path}: no metric `{only}`"); + } + ensure!( + !metrics.is_empty(), + "{path}: workloads[0].e2e.metrics is empty" + ); + let samples_per_sec = match samples_per_sec { + Some(rate) => rate, + // One feeder sends every metric at one rate. + None if rates.iter().all(|&r| r == rates[0]) => rates[0], + None => bail!( + "metrics have different samples_per_sec_per_series ({rates:?}); \ + send them with separate feeders (--metric) or set --samples-per-sec" + ), + }; + Ok(Workload { + samples_per_sec, + metrics, + }) +} + +fn metric(name: &str, spec: &Value, seed: u64) -> Result<(Metric, u32)> { + ensure!( + is_identifier(name), + "the name must match [A-Za-z_][A-Za-z0-9_]* (it is also the ClickHouse table)" + ); + let kind = spec.get("type").and_then(Value::as_str).unwrap_or_default(); + if kind != "gauge" && kind != "counter" { + bail!("type `{kind}` is not supported; the feeder generates `gauge` and `counter` metrics"); + } + let spec: MetricSpec = serde_json::from_value(spec.clone())?; + let series = series_labels(&spec.labels)?; + let (skew, kind) = if spec.kind == "gauge" { + let values: GaugeValues = serde_json::from_value(spec.values).context("gauge `values`")?; + ensure!( + values.dist.pareto > 0.0, + "the Pareto shape must be positive" + ); + let kind = Kind::Gauge { + pareto_shape: values.dist.pareto, + pareto_scale: 1.0, + }; + (values.value, kind) + } else { + let values: CounterValues = + serde_json::from_value(spec.values).context("counter `values`")?; + (values.increment, Kind::Counter) + }; + ensure!( + skew.keys == series.len() as u64, + "Zipf keys ({}) must equal the number of series ({}): each series is a key", + skew.keys, + series.len() + ); + ensure!(skew.zipf >= 0.0, "the Zipf skew must be non-negative"); + // The ranks depend on the seed alone, so a key is equally heavy in every + // metric of the table. + let scales = zipf_scales(series.len(), skew.zipf, seed); + let metric = Metric { + name: name.to_string(), + labels: spec.labels, + series, + scales, + kind, + seed: splitmix64(seed ^ fnv1a(name)), + }; + Ok((metric, spec.samples_per_sec_per_series)) +} + +/// Each series' label values, in `labels` order. Series are numbered in mixed +/// radix with the last label varying fastest. +pub fn series_labels(labels: &[LabelSpec]) -> Result>> { + ensure!(!labels.is_empty(), "a metric needs at least one label"); + // Index of each label's parent, which must come earlier. + let mut parents = Vec::with_capacity(labels.len()); + for (i, label) in labels.iter().enumerate() { + let name = &label.name; + ensure!( + is_identifier(name) && !name.starts_with("__"), + "label `{name}` must match [A-Za-z_][A-Za-z0-9_]* and not start with `__`" + ); + ensure!( + name != "ts" && name != "value", + "label `{name}` clashes with a ClickHouse column" + ); + ensure!( + labels[..i].iter().all(|l| &l.name != name), + "label `{name}` is repeated" + ); + ensure!(label.cardinality > 0, "label `{name}` has cardinality 0"); + parents.push(match &label.child_of { + None => None, + Some(parent) => Some( + labels[..i] + .iter() + .position(|l| &l.name == parent) + .with_context(|| { + format!("label `{name}`: child_of `{parent}` must name an earlier label") + })?, + ), + }); + } + let num_series = labels + .iter() + .try_fold(1usize, |n, l| { + n.checked_mul(usize::try_from(l.cardinality).ok()?) + }) + .context("too many series")?; + + let mut series = Vec::with_capacity(num_series); + let mut index = vec![0u64; labels.len()]; + for _ in 0..num_series { + let mut values: Vec = Vec::with_capacity(labels.len()); + for (i, parent) in parents.iter().enumerate() { + values.push(match parent { + None => index[i].to_string(), + Some(p) => format!("{}.{}", values[*p], index[i]), + }); + } + series.push(values); + // Advance the mixed-radix index, last label fastest. + for i in (0..labels.len()).rev() { + index[i] += 1; + if index[i] < labels[i].cardinality { + break; + } + index[i] = 0; + } + } + Ok(series) +} + +/// Each series' value scale for Zipf(`theta`) skew over the series: the +/// series of rank `r` (0 is the heaviest) gets `(r + 1)^-theta`, normalized so +/// the scales average 1. Ranks are a seeded shuffle of the series, so the +/// heavy series spread across parent labels instead of filling the first one. +pub fn zipf_scales(num_series: usize, theta: f64, seed: u64) -> Vec { + let weights: Vec = (0..num_series) + .map(|r| ((r + 1) as f64).powf(-theta)) + .collect(); + let norm = num_series as f64 / weights.iter().sum::(); + zipf_ranks(num_series, seed) + .into_iter() + .map(|r| weights[r] * norm) + .collect() +} + +/// A seeded permutation: the Zipf rank of each series (Fisher-Yates). +pub fn zipf_ranks(num_series: usize, seed: u64) -> Vec { + let mut ranks: Vec = (0..num_series).collect(); + let mut state = splitmix64(seed ^ fnv1a("zipf ranks")); + for i in (1..num_series).rev() { + state = splitmix64(state); + ranks.swap(i, (state % (i as u64 + 1)) as usize); + } + ranks +} + +/// One row of the facts check: the distinct value tuples of a label subset. +#[derive(Debug, Clone, PartialEq)] +pub struct Fact { + pub labels: Vec, + /// The product of the fan-outs of `labels` closed under `child_of`. + pub expected: u64, + /// Distinct tuples among the generated series. + pub actual: u64, +} + +/// For every non-empty subset of `labels`, the expected and the generated +/// number of distinct label-value tuples. +pub fn facts(labels: &[LabelSpec], series: &[Vec]) -> Vec { + let n = labels.len(); + assert!(n < 32, "too many labels to enumerate their subsets"); + (1u32..1 << n) + .map(|mask| { + let subset: Vec = (0..n).filter(|i| mask & (1 << i) != 0).collect(); + let actual = series + .iter() + .map(|values| { + subset + .iter() + .map(|&i| values[i].as_str()) + .collect::>() + }) + .collect::>() + .len() as u64; + Fact { + labels: subset.iter().map(|&i| labels[i].name.clone()).collect(), + expected: closed_cardinality(labels, &subset), + actual, + } + }) + .collect() +} + +/// The product of the cardinalities of `subset` and every ancestor of its +/// labels: a child value names its parent, so grouping by a child groups by +/// its ancestors too. +fn closed_cardinality(labels: &[LabelSpec], subset: &[usize]) -> u64 { + let mut closed = HashSet::new(); + let mut todo: Vec = subset.to_vec(); + while let Some(i) = todo.pop() { + if closed.insert(i) { + if let Some(parent) = &labels[i].child_of { + todo.extend(labels.iter().position(|l| &l.name == parent)); + } + } + } + closed + .iter() + .map(|&i| labels[i].cardinality) + .fold(1u64, u64::saturating_mul) +} + +fn is_identifier(name: &str) -> bool { + let mut chars = name.chars(); + chars + .next() + .is_some_and(|c| c.is_ascii_alphabetic() || c == '_') + && chars.all(|c| c.is_ascii_alphanumeric() || c == '_') +} + +/// 64-bit FNV-1a, to derive a per-name seed. +fn fnv1a(s: &str) -> u64 { + s.bytes().fold(0xcbf2_9ce4_8422_2325, |h, b| { + (h ^ u64::from(b)).wrapping_mul(0x0000_0100_0000_01b3) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn label(name: &str, cardinality: u64, child_of: Option<&str>) -> LabelSpec { + LabelSpec { + name: name.to_string(), + cardinality, + child_of: child_of.map(str::to_string), + } + } + + fn classic_labels() -> Vec { + vec![label("job", 10, None), label("label_0", 100, Some("job"))] + } + + #[test] + fn child_labels_fan_out_under_their_parent() { + let series = series_labels(&classic_labels()).unwrap(); + assert_eq!(series.len(), 1_000); + assert_eq!(series[0], ["0", "0.0"]); + assert_eq!(series[1], ["0", "0.1"]); + assert_eq!(series[100], ["1", "1.0"]); + assert_eq!(series[999], ["9", "9.99"]); + // A child value names its parent, so it is unique across parents. + let children: HashSet<&str> = series.iter().map(|s| s[1].as_str()).collect(); + assert_eq!(children.len(), 1_000); + assert!(series + .iter() + .all(|s| s[1].starts_with(&format!("{}.", s[0])))); + } + + #[test] + fn root_labels_form_a_cross_product() { + let labels = vec![ + label("region", 2, None), + label("service", 3, None), + label("endpoint", 2, Some("service")), + ]; + let series = series_labels(&labels).unwrap(); + assert_eq!(series.len(), 12); + let pairs: HashSet<(&str, &str)> = series + .iter() + .map(|s| (s[0].as_str(), s[1].as_str())) + .collect(); + assert_eq!(pairs.len(), 6); + assert_eq!(series[11], ["1", "2", "2.1"]); + } + + #[test] + fn rejects_a_child_before_its_parent() { + let labels = vec![label("label_0", 3, Some("job")), label("job", 2, None)]; + assert!(series_labels(&labels).is_err()); + } + + #[test] + fn facts_hold_on_generated_series() { + let labels = vec![ + label("region", 4, None), + label("service", 5, None), + label("endpoint", 3, Some("service")), + label("status", 2, None), + ]; + let series = series_labels(&labels).unwrap(); + let facts = facts(&labels, &series); + assert_eq!(facts.len(), 15); + assert!(facts.iter().all(|f| f.expected == f.actual), "{facts:?}"); + let by = |names: &[&str]| facts.iter().find(|f| f.labels == names).unwrap().expected; + // `endpoint` alone groups by its service too. + assert_eq!(by(&["endpoint"]), 15); + assert_eq!(by(&["region", "endpoint"]), 60); + assert_eq!(by(&["region", "service", "endpoint", "status"]), 120); + } + + #[test] + fn facts_catch_a_wrong_expectation() { + let labels = classic_labels(); + let series = series_labels(&labels).unwrap(); + // Claiming label_0 is a cross product rather than a fan-out expects + // 100 label_0 values, but the generated series have 1000. + let wrong = vec![label("job", 10, None), label("label_0", 100, None)]; + let facts = facts(&wrong, &series); + let label_0 = facts.iter().find(|f| f.labels == ["label_0"]).unwrap(); + assert_eq!((label_0.expected, label_0.actual), (100, 1_000)); + } + + #[test] + fn zipf_scales_average_one_and_spread_the_heavy_series() { + let scales = zipf_scales(1_000, 1.1, 42); + let mean = scales.iter().sum::() / 1_000.0; + assert!((mean - 1.0).abs() < 1e-9, "mean {mean}"); + + let ranks = zipf_ranks(1_000, 42); + let mut sorted = ranks.clone(); + sorted.sort_unstable(); + assert!(sorted.into_iter().eq(0..1_000)); + // Scales decrease with rank. + let by_rank = |r: usize| scales[ranks.iter().position(|&x| x == r).unwrap()]; + assert!((1..1_000).all(|r| by_rank(r) < by_rank(r - 1))); + // The 10 heaviest series fall in several of the 10 jobs (100 series + // each), not all in the first. + let jobs: HashSet = (0..1_000) + .filter(|&s| ranks[s] < 10) + .map(|s| s / 100) + .collect(); + assert!(jobs.len() >= 5, "{jobs:?}"); + // The ranks depend on the seed. + assert_ne!(ranks, zipf_ranks(1_000, 43)); + } + + fn write_table(metrics: Value) -> String { + let path = std::env::temp_dir().join(format!( + "paced-feeder-table-{}-{}.json", + std::process::id(), + fnv1a(&metrics.to_string()) + )); + let table = json!({"workloads": [{"dataset": "x", "e2e": {"metrics": metrics}}]}); + std::fs::write(&path, table.to_string()).unwrap(); + path.to_string_lossy().into_owned() + } + + fn classic_metrics() -> Value { + let labels = json!([{"name": "job", "cardinality": 2}, + {"name": "label_0", "cardinality": 3, "child_of": "job"}]); + let keys = json!({"zipf": 1.1, "keys": 6}); + json!({ + "data": {"type": "gauge", "labels": labels, "samples_per_sec_per_series": 4, + "values": {"value": keys, "dist": {"pareto": 2.0}}}, + "data_total": {"type": "counter", "labels": labels, "samples_per_sec_per_series": 4, + "values": {"increment": keys}}, + }) + } + + #[test] + fn loads_gauge_and_counter_from_a_table() { + let path = write_table(classic_metrics()); + let w = load(&path, None, None, 1).unwrap(); + assert_eq!(w.samples_per_sec, 4); + assert_eq!(w.num_series(), 12); + let (data, total) = (&w.metrics[0], &w.metrics[1]); + assert_eq!( + (data.name.as_str(), total.name.as_str()), + ("data", "data_total") + ); + assert_eq!( + data.kind, + Kind::Gauge { + pareto_shape: 2.0, + pareto_scale: 1.0 + } + ); + assert_eq!(total.kind, Kind::Counter); + // The same series is heavy in both, but the gauges' draws differ by metric. + assert_eq!(data.scales, total.scales); + assert_ne!(data.seed, total.seed); + + let only = load(&path, Some("data_total"), Some(2), 1).unwrap(); + assert_eq!(only.metrics.len(), 1); + assert_eq!(only.samples_per_sec, 2); + assert!(load(&path, Some("nope"), None, 1).is_err()); + } + + #[test] + fn rejects_records_and_mismatched_keys() { + let path = write_table(json!({"http": {"type": "records", "labels": [], "rate": 1}})); + let err = format!("{:#}", load(&path, None, None, 1).unwrap_err()); + assert!(err.contains("type `records` is not supported"), "{err}"); + + let mut metrics = classic_metrics(); + metrics["data"]["values"]["value"]["keys"] = json!(7); + let path = write_table(metrics); + let err = format!("{:#}", load(&path, Some("data"), None, 1).unwrap_err()); + assert!(err.contains("must equal the number of series"), "{err}"); + } +} diff --git a/asap-tools/data-sources/paced-feeder/src/workload.rs b/asap-tools/data-sources/paced-feeder/src/workload.rs index 3b93561e..de8f5d31 100644 --- a/asap-tools/data-sources/paced-feeder/src/workload.rs +++ b/asap-tools/data-sources/paced-feeder/src/workload.rs @@ -1,11 +1,13 @@ -//! Seeded synthetic workload: `groups × series_per_group` series, each emitting -//! `samples_per_sec` Pareto-distributed values per second. +//! Seeded synthetic workload: one or more metrics, each a set of labelled +//! series emitting `samples_per_sec` evenly spaced samples per second. //! -//! A sample's value depends only on `(seed, series, sample index)`, never on -//! wall-clock time or on which sink sends it, so two feeders started with the -//! same parameters produce identical data. +//! A sample's value depends only on `(metric seed, series, sample index)`, +//! never on wall-clock time or on which sink sends it, so two feeders started +//! with the same parameters produce identical data. -/// One generated sample. `series` indexes into [`Workload::series_labels`]. +use serde::Deserialize; + +/// One generated sample of one metric. `series` indexes into [`Metric::series`]. #[derive(Debug, Clone, Copy, PartialEq)] pub struct Row { pub series: usize, @@ -13,56 +15,137 @@ pub struct Row { pub value: f64, } +/// A label with `cardinality` values, or, with `child_of`, that many values +/// under each value of the parent label. +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LabelSpec { + pub name: String, + pub cardinality: u64, + #[serde(default)] + pub child_of: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Kind { + /// `scale · pareto_scale · U^(-1/pareto_shape)`: Pareto(shape, scale) times + /// the series' scale. + Gauge { + pareto_shape: f64, + pareto_scale: f64, + }, + /// `(n + 1) · scale` at sample index `n`: starts at the first sample, grows + /// by the series' scale per sample and never resets. + Counter, +} + #[derive(Debug, Clone)] -pub struct Workload { +pub struct Metric { + pub name: String, + pub labels: Vec, + /// Each series' label values, in `labels` order. + pub series: Vec>, + /// Each series' value multiplier. + pub scales: Vec, + pub kind: Kind, pub seed: u64, - pub groups: usize, - pub series_per_group: usize, +} + +#[derive(Debug, Clone)] +pub struct Workload { pub samples_per_sec: u32, - pub pareto_shape: f64, - pub pareto_scale: f64, + pub metrics: Vec, } impl Workload { + /// The fixed `--groups C × --series-per-group s` workload: one gauge with + /// series `g * s + i` labelled `label_0="g{g}", instance="i{i}"`. + pub fn grouped( + name: &str, + seed: u64, + groups: usize, + series_per_group: usize, + samples_per_sec: u32, + pareto_shape: f64, + pareto_scale: f64, + ) -> Self { + let series: Vec> = (0..groups * series_per_group) + .map(|s| { + vec![ + format!("g{}", s / series_per_group), + format!("i{}", s % series_per_group), + ] + }) + .collect(); + let scales = vec![1.0; series.len()]; + let label = |name: &str, cardinality: usize| LabelSpec { + name: name.to_string(), + cardinality: cardinality as u64, + child_of: None, + }; + let metric = Metric { + name: name.to_string(), + labels: vec![ + label("label_0", groups), + label("instance", series_per_group), + ], + series, + scales, + kind: Kind::Gauge { + pareto_shape, + pareto_scale, + }, + seed, + }; + Self { + samples_per_sec, + metrics: vec![metric], + } + } + pub fn num_series(&self) -> usize { - self.groups * self.series_per_group + self.metrics.iter().map(|m| m.series.len()).sum() } pub fn rows_per_tick(&self) -> usize { self.num_series() * self.samples_per_sec as usize } - /// `(label_0, instance)` for a series: series `g * series_per_group + i` - /// is `("g{g}", "i{i}")`. - pub fn series_labels(&self, series: usize) -> (String, String) { - let group = series / self.series_per_group; - let instance = series % self.series_per_group; - (format!("g{group}"), format!("i{instance}")) - } - - /// Samples for sub-second slot `slot` of one-second tick `tick`, one per - /// series. Slots are spaced evenly over `(t, t + 1 s]`, where `t` is the - /// tick's start: the same left-open interval as ASAPQuery's one-second - /// windows, so a tick never spans two of them. - pub fn slot_rows(&self, tick: u64, slot: u32, start_ms: i64) -> Vec { + /// Samples of `metric` for sub-second slot `slot` of one-second tick + /// `tick`, one per series. Slots are spaced evenly over `(t, t + 1 s]`, + /// where `t` is the tick's start: the same left-open interval as + /// ASAPQuery's one-second windows, so a tick never spans two of them. + pub fn slot_rows(&self, metric: usize, tick: u64, slot: u32, start_ms: i64) -> Vec { let per_sec = u64::from(self.samples_per_sec); let slot = u64::from(slot); let timestamp_ms = start_ms + tick as i64 * 1_000 + ((slot + 1) * 1_000 / per_sec) as i64; let sample_index = tick * per_sec + slot; - (0..self.num_series()) + let metric = &self.metrics[metric]; + (0..metric.series.len()) .map(|series| Row { series, timestamp_ms, - value: self.value(series, sample_index), + value: metric.value(series, sample_index), }) .collect() } +} - /// Pareto(shape, scale) by inverse transform: `scale · U^(-1/shape)` with - /// `U` uniform in (0, 1], so every value is ≥ `scale` and positive. +impl Metric { pub fn value(&self, series: usize, sample_index: u64) -> f64 { - let u = unit_uniform(self.seed, series as u64, sample_index); - self.pareto_scale * u.powf(-1.0 / self.pareto_shape) + let scale = self.scales[series]; + match self.kind { + // Pareto by inverse transform with `U` uniform in (0, 1], so every + // value is ≥ `scale · pareto_scale` and positive. + Kind::Gauge { + pareto_shape, + pareto_scale, + } => { + let u = unit_uniform(self.seed, series as u64, sample_index); + scale * pareto_scale * u.powf(-1.0 / pareto_shape) + } + Kind::Counter => (sample_index + 1) as f64 * scale, + } } } @@ -74,7 +157,7 @@ fn unit_uniform(seed: u64, series: u64, sample_index: u64) -> f64 { ((x >> 11) + 1) as f64 / (1u64 << 53) as f64 } -fn splitmix64(mut z: u64) -> u64 { +pub fn splitmix64(mut z: u64) -> u64 { z = z.wrapping_add(0x9E37_79B9_7F4A_7C15); z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); @@ -86,21 +169,14 @@ mod tests { use super::*; fn workload() -> Workload { - Workload { - seed: 7, - groups: 3, - series_per_group: 2, - samples_per_sec: 2, - pareto_shape: 1.5, - pareto_scale: 1.0, - } + Workload::grouped("data", 7, 3, 2, 2, 1.5, 1.0) } #[test] fn each_slot_has_every_series_at_its_timestamp() { let w = workload(); for (slot, timestamp_ms) in [(0, 15_500), (1, 16_000)] { - let rows = w.slot_rows(5, slot, 10_000); + let rows = w.slot_rows(0, 5, slot, 10_000); assert_eq!(rows.len(), 6); assert!(rows.iter().all(|r| r.timestamp_ms == timestamp_ms)); assert!(rows.iter().map(|r| r.series).eq(0..6)); @@ -110,43 +186,42 @@ mod tests { #[test] fn values_depend_only_on_seed_series_and_sample_index() { let w = workload(); - // Same slot, different start time: same values. - let a: Vec = w.slot_rows(4, 1, 0).iter().map(|r| r.value).collect(); - let b: Vec = w - .slot_rows(4, 1, 1_000_000) - .iter() - .map(|r| r.value) - .collect(); - assert_eq!(a, b); - // A different seed changes them. - let other = Workload { seed: 8, ..w }; - assert_ne!( - a, - other - .slot_rows(4, 1, 0) + let values = |w: &Workload, start_ms| -> Vec { + w.slot_rows(0, 4, 1, start_ms) .iter() .map(|r| r.value) - .collect::>() - ); + .collect() + }; + // Same slot, different start time: same values. + assert_eq!(values(&w, 0), values(&w, 1_000_000)); + // A different seed changes them. + let other = Workload::grouped("data", 8, 3, 2, 2, 1.5, 1.0); + assert_ne!(values(&w, 0), values(&other, 0)); + } + + #[test] + fn grouped_values_keep_the_seeded_pareto_draw() { + // The grouped workload's values are the plain Pareto draw of + // `(seed, series, sample index)`, as before metrics had scales. + let w = workload(); + let u = unit_uniform(7, 3, 9); + assert_eq!(w.metrics[0].value(3, 9), u.powf(-1.0 / 1.5)); } #[test] fn labels_split_series_into_groups_and_instances() { let w = workload(); - assert_eq!(w.series_labels(0), ("g0".to_string(), "i0".to_string())); - assert_eq!(w.series_labels(3), ("g1".to_string(), "i1".to_string())); - assert_eq!(w.series_labels(5), ("g2".to_string(), "i1".to_string())); + let series = &w.metrics[0].series; + assert_eq!(series[0], ["g0", "i0"]); + assert_eq!(series[3], ["g1", "i1"]); + assert_eq!(series[5], ["g2", "i1"]); } #[test] fn values_follow_pareto_distribution() { - let w = Workload { - groups: 1, - series_per_group: 1, - ..workload() - }; + let w = Workload::grouped("data", 7, 1, 1, 2, 1.5, 1.0); let n = 200_000u64; - let mut values: Vec = (0..n).map(|i| w.value(0, i)).collect(); + let mut values: Vec = (0..n).map(|i| w.metrics[0].value(0, i)).collect(); assert!(values.iter().all(|&v| v >= 1.0)); values.sort_by(f64::total_cmp); // Pareto(a, xm) quantile: xm · (1 - q)^(-1/a). @@ -159,4 +234,18 @@ mod tests { ); } } + + #[test] + fn counters_increase_strictly_per_series() { + let metric = Metric { + kind: Kind::Counter, + scales: vec![0.5, 2.0], + ..workload().metrics[0].clone() + }; + for series in 0..2 { + let values: Vec = (0..100).map(|n| metric.value(series, n)).collect(); + assert_eq!(values[0], metric.scales[series]); + assert!(values.windows(2).all(|w| w[1] > w[0])); + } + } } From f3daa66fc459a6d21f23413584a6b099b35bc76e Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 13 Oct 2026 01:46:20 +0000 Subject: [PATCH 3/8] feat(asap-tools): generate records metrics in the paced feeder `--workload-table` now also sends a table's `records` metrics (the multi-grouping `http` and `flows`). Each record is a stateless hash of the seed, the metric and its index: labels are Zipf(`skew`) draws (a child given its parent, value `i` of rank `i`), keys are Zipf ranks, a burst sends `share` of the records to its label value from its own keys, and Pareto values are scaled by their `scaled_by` label's factor (aqpbm-datagen's log-uniform [1, 10]) and by the anomaly factor on the anomaly label's first value. A tick starting at t spreads its records over 10 ms slots in (t, t + 1 s], slot s at t + (s + 1) * 10 ms, and is sent concurrently like gauge/counter ticks, so one feeder can send both kinds. Records are generated inside the encoding tasks. Remote write sends one sample per value field, named `.`; ClickHouse gets one row per record in a table named after the metric. `--records-per-sec` scales a run down, and `--verify-facts` also compares the shares of each label's heaviest values and of the burst with sketch-bench's `label_shares()`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AnJ7LocWxguh3q4QWkCYae --- .../data-sources/paced-feeder/README.md | 119 ++- .../data-sources/paced-feeder/src/encode.rs | 434 ++++++++- .../data-sources/paced-feeder/src/main.rs | 214 +++-- .../data-sources/paced-feeder/src/records.rs | 880 ++++++++++++++++++ .../data-sources/paced-feeder/src/table.rs | 216 ++++- .../data-sources/paced-feeder/src/workload.rs | 77 +- 6 files changed, 1779 insertions(+), 161 deletions(-) create mode 100644 asap-tools/data-sources/paced-feeder/src/records.rs diff --git a/asap-tools/data-sources/paced-feeder/README.md b/asap-tools/data-sources/paced-feeder/README.md index c7362364..fea7a119 100644 --- a/asap-tools/data-sources/paced-feeder/README.md +++ b/asap-tools/data-sources/paced-feeder/README.md @@ -15,7 +15,8 @@ the same stream without a dataset file on disk. The feeder generates either the fixed `--groups` workload or the metrics of a workload table (`--workload-table`). Either way, a value depends only on -`(--seed, metric, series, sample index)`, never on wall-clock time or the sink, +`(--seed, metric, series, sample index)` (a record only on +`(--seed, metric, record index)`), never on wall-clock time or the sink, and each series emits `--samples-per-sec` samples per second, evenly spaced. ### `--groups` workload @@ -63,13 +64,69 @@ metric in it, or only `--metric NAME`. For example: sample and never resetting, so `rate()` is `scale_i · samples/s`. - `--samples-per-sec` overrides `samples_per_sec_per_series`, e.g. to scale a run down. Without it, the sent metrics must share one rate. -- Other metric types (the multi-grouping `records` metrics) are rejected. -`--verify-facts` checks the generated series against the description without +#### `records` metrics + +The multi-grouping templates' metrics (`http`, `flows`, or `data_3_http` with +several metrics) are streams of records, each with a value of every label and +of every value field: + +```json +"http": {"type": "records", "records_per_sec": 2000000, + "labels": [{"name": "region", "cardinality": 4, "skew": 0.5}, + {"name": "service", "cardinality": 25, "skew": 1.1}, + {"name": "endpoint", "cardinality": 25, "child_of": "service", "skew": 1.1}, + {"name": "status", "cardinality": 4, "skew": 2.0}], + "values": {"latency": {"pareto": 2.0, "scaled_by": "service", + "anomaly": {"label": "endpoint", "factor": 10}}, + "user_id": {"zipf": 0.8, "keys": 1000000}}}, +"flows": {"type": "records", "records_per_sec": 4000000, + "labels": [{"name": "dst_subnet", "cardinality": 1000, "skew": 1.1}, …], + "values": {"src_ip": {"zipf": 0.0, "keys": 1000000, + "burst": {"label": "dst_subnet", "value": 999, + "share": 0.05, "keys": 10000}}}} +``` + +Every draw of record `n` is a hash of the metric's seed (from `--seed` and its +name), `n` and the draw, so records match sketch-bench's `label_shares()` +(`scripts/export_autosketch_eval_table.py`), which the optimizer assumed: + +- **Labels** are drawn independently, a child given its parent: value `i` (a + child's index `i` under its parent) has weight `(i+1)^-skew`, so `0` is the + heaviest. Values are named as for gauges (`service="3"`, + `endpoint="3.17"`). +- **Keys** (`zipf` θ over `keys`): the key of Zipf rank `r` is `r`, so key 0 is + the most frequent. +- **Burst**: with probability `share` a record is a burst record: its `label` + is `value` (a child index for a child label) and its key is uniform over + `keys` burst keys numbered after the background keys (`1000000` to + `1009999` for `flows`). Other records draw both normally, so `dst_subnet`'s + shares are Zipf times `1 - share`, plus `share` at `999`. +- **Pareto** (`latency`): `1000 · U^(-1/a)` (`U` in (0, 1]), times its service's + fixed factor with `scaled_by`, times `factor` on the anomaly label's first + value (`endpoint="0.0"`, the heaviest endpoint of the heaviest service). A + service's factor is aqpbm-datagen's `scale_factor` (as in sketch-bench's + `configs/datagen/hydra_http_latency.yaml`, scale 1000 and `scale_range: + [1.0, 10.0]`): `10^u` with `u` in [0, 1) a SplitMix64 hash of the value's + FNV-1a xor `--seed`, so it is the same in every metric of a table. Scaling + keeps the tail index `a`. +- **Time.** A tick starting at `t` spreads its `records_per_sec` records + evenly over 100 slots of 10 ms; slot `s` has every record at + `t + (s + 1) · 10 ms`, so the tick lies in `(t, t + 1 s]`. +- `--records-per-sec` overrides every records metric's rate, e.g. to scale a + run down. + +`--verify-facts` checks the generated data against the description without sending anything: for every non-empty subset of a metric's labels it prints the expected number of distinct label-value tuples (the product of the cardinalities of the subset plus every label's `child_of` ancestors) and the -number among the generated series, and exits non-zero if any differ. +number among the generated series (for records metrics, among all label-value +combinations a record can have), and exits non-zero if any differ. For records +metrics it also draws records `0..1e6` and compares each label's three +heaviest values' shares (a child's index pooled over parents, i.e. its share +given the parent), the burst value's share and the share of burst records with +`label_shares()`; a share fails if it is off by more than 5 standard errors, +`5·sqrt(p(1-p)/n)`. ```sh $ paced_feeder --workload-table synthetic-templatesclassic-shared1.json --verify-facts @@ -80,6 +137,24 @@ data: 10000 series job,label_0 10000 10000 data_total: 10000 series … + +$ paced_feeder --workload-table synthetic-templatesmultigroup-shared1.json --verify-facts +flows: 4000000 records/s, 3000000 label combinations + labels expected generated + dst_subnet 1000 1000 + … + shares of 1000000 records (tolerance 5 standard errors) + label value expected generated tolerance + dst_subnet 0 0.17047 0.17069 0.00188 + … + dst_subnet 999 0.05009 0.05037 0.00109 + … + burst records 0.05000 0.05029 0.00109 + burst keys: 9938 distinct of 10000 +http: 2000000 records/s, 10000 label combinations + … + endpoint *.0 (given service) 0.29841 0.29855 0.00229 + … ``` Event time starts at the next whole wall-clock second (or `--start-ms`), so @@ -112,6 +187,14 @@ Remote write sends each metric's name as `__name__` and its labels sorted by name, as Prometheus requires; it also works against Prometheus' remote-write receiver (`--web.enable-remote-write-receiver`). +A record of a records metric becomes one sample per value field, named +`.` (`http.latency`, `http.user_id`), with the record's labels +and timestamp. ASAPQuery's SQL mode routes a sample to the aggregations whose +metric is `table.value_column` (or whose table is the sample's name), so each +field feeds only its own aggregations. Many records of a slot share a series +and timestamp, so Prometheus, which keeps one sample per series and +timestamp, is not a target for records metrics. + With `--groups`, the ClickHouse table (`--table`) must have these columns (any engine and ordering): @@ -131,6 +214,19 @@ CREATE TABLE data_total (ts DateTime64(3), job String, label_0 String, value Flo ENGINE = MergeTree ORDER BY (job, label_0, ts); ``` +A records metric goes to the table named after it, with columns `ts`, the +labels in the description's order, then the value fields in name order (keys +are integers below 2^53, exact in `Float64`): + +```sql +CREATE TABLE http (ts DateTime64(3), region String, service String, + endpoint String, status String, latency Float64, user_id Float64) +ENGINE = MergeTree ORDER BY (region, service, endpoint, status, ts); +CREATE TABLE flows (ts DateTime64(3), dst_subnet String, dst_port String, + proto String, src_ip Float64) +ENGINE = MergeTree ORDER BY (dst_subnet, dst_port, proto, ts); +``` + `CLICKHOUSE_USER` / `CLICKHOUSE_PASSWORD` (or the matching flags) set credentials. @@ -150,6 +246,11 @@ default batch size is 50,000 rows for remote write, which keeps a compressed request under axum's 2 MB body limit in ASAPQuery's ingest, and 100,000 for ClickHouse, where every INSERT creates a part. +For records metrics, each request generates its own records as it encodes +them, so generation runs on all cores. For remote write, `--batch-rows` +counts samples, a record being one per value field (`http`: 25,000 records +per request). Rows in the stats are records. + `--stats-out` writes, per tick, the rows, bytes, start lag and elapsed time, plus totals: - `achieved_rows_per_sec`; @@ -168,3 +269,13 @@ Measured on a CloudLab c6320 (feeder pinned to 2 cores, 10 s at 1e6 rows/s): |---|---|---| | ClickHouse 26.10 | 0 | 0.15 cores | | ASAPQuery precompute engine | 0 | 0.34 cores | + +Records metrics at full rate against a local HTTP stub that discards the +bodies (shared, loaded 56-core machine, default flags, mean tick time over +10 s, no overruns): + +| Metric | Remote write | ClickHouse | +|---|---|---| +| `http` (2e6 records/s, 4e6 samples/s) | 171 ms, 2.5 cores | 104 ms, 1.1 cores | +| `flows` (4e6 records/s) | 195 ms, 2.8 cores | 117 ms, 1.5 cores | +| both (6e6 records/s) | 340 ms, 4.9 cores | 194 ms, 2.5 cores | diff --git a/asap-tools/data-sources/paced-feeder/src/encode.rs b/asap-tools/data-sources/paced-feeder/src/encode.rs index bb6249b2..9cda3334 100644 --- a/asap-tools/data-sources/paced-feeder/src/encode.rs +++ b/asap-tools/data-sources/paced-feeder/src/encode.rs @@ -1,11 +1,30 @@ -//! Wire encoders for the two sinks. Each series' labels are encoded once up -//! front, so the per-sample cost at high rates is just the value and timestamp. +//! Wire encoders for the two sinks. Each series' labels (or each label value +//! of a records metric) are encoded once up front, so the per-sample cost at +//! high rates is just the values and timestamp. -use crate::workload::{Metric, Row}; +use crate::records::{RecordMetric, Records}; +use crate::workload::{Metric, Row, Rows}; use prost::encoding::{encode_key, encode_varint, encoded_len_varint, WireType}; +use std::ops::Range; +use std::sync::Arc; pub trait Encoder: Send + Sync { - fn encode(&self, rows: &[Row]) -> Vec; + /// Encodes `rows[range]`, which are of the kind the encoder was built for. + fn encode(&self, rows: &Rows, range: Range) -> Vec; +} + +fn samples(rows: &Rows) -> &[Row] { + match rows { + Rows::Samples(rows) => rows, + Rows::Records(_) => panic!("a series encoder was given records"), + } +} + +fn records(rows: &Rows) -> &Records { + match rows { + Rows::Records(records) => records, + Rows::Samples(_) => panic!("a records encoder was given samples"), + } } /// Prometheus remote write: a snappy-compressed `WriteRequest` with one @@ -33,10 +52,7 @@ impl RemoteWriteEncoder { labels.sort_unstable(); let mut buf = Vec::new(); for (name, value) in labels { - let mut label = Vec::new(); - encode_bytes_field(1, name.as_bytes(), &mut label); - encode_bytes_field(2, value.as_bytes(), &mut label); - encode_bytes_field(1, &label, &mut buf); + encode_label(name, value, &mut buf); } buf }) @@ -46,26 +62,125 @@ impl RemoteWriteEncoder { } impl Encoder for RemoteWriteEncoder { - fn encode(&self, rows: &[Row]) -> Vec { + fn encode(&self, rows: &Rows, range: Range) -> Vec { let mut request = Vec::new(); let mut series = Vec::new(); - let mut sample = Vec::new(); - for row in rows { - sample.clear(); - encode_key(1, WireType::SixtyFourBit, &mut sample); - sample.extend_from_slice(&row.value.to_le_bytes()); - encode_key(2, WireType::Varint, &mut sample); - encode_varint(row.timestamp_ms as u64, &mut sample); - + for row in &samples(rows)[range] { series.clear(); series.extend_from_slice(&self.series_labels[row.series]); - encode_bytes_field(2, &sample, &mut series); - + encode_sample(row.value, row.timestamp_ms, &mut series); encode_bytes_field(1, &series, &mut request); } - snap::raw::Encoder::new() - .compress_vec(&request) - .expect("snappy compression of an in-memory buffer cannot fail") + compress(&request) + } +} + +/// Generates records `range` of `rows` (indices relative to their first) and +/// calls `f` with each one's timestamp, label value indices and values. +fn for_each_record( + metric: &RecordMetric, + rows: &Rows, + range: Range, + mut f: impl FnMut(i64, &[u32], &[f64]), +) { + let records = records(rows); + let mut labels = vec![0u32; metric.labels.len()]; + let mut values = vec![0.0; metric.fields.len()]; + let first = records.indices.start; + for index in first + range.start as u64..first + range.end as u64 { + metric.record(index, &mut labels, &mut values); + f( + metric.timestamp_ms(index, records.start_ms), + &labels, + &values, + ); + } +} + +/// Remote write of a records metric: each record is one sample per value +/// field, named `.`, with the record's labels and timestamp. +/// ASAPQuery's SQL mode routes a sample to the aggregations of +/// `table.value_column`, so each field feeds only its own aggregations. +pub struct RecordsRemoteWriteEncoder { + metric: Arc, + /// The labels in name order, `None` for `__name__`. + order: Vec>, + /// The encoded `Label` field of each value of each label. + label_values: Vec>>, + /// The encoded `__name__` `Label` field of each value field. + names: Vec>, +} + +/// The `__name__` of value field `field` of records metric `metric`. +pub fn record_sample_name(metric: &str, field: &str) -> String { + format!("{metric}.{field}") +} + +impl RecordsRemoteWriteEncoder { + pub fn new(metric: Arc) -> Self { + let mut sorted: Vec<(&str, Option)> = metric + .labels + .iter() + .enumerate() + .map(|(i, l)| (l.name.as_str(), Some(i))) + .collect(); + sorted.push(("__name__", None)); + sorted.sort_unstable(); + let label_values = metric + .labels + .iter() + .zip(&metric.label_values) + .map(|(label, values)| { + values + .iter() + .map(|value| { + let mut buf = Vec::new(); + encode_label(&label.name, value, &mut buf); + buf + }) + .collect() + }) + .collect(); + let names = metric + .fields + .iter() + .map(|field| { + let mut buf = Vec::new(); + encode_label( + "__name__", + &record_sample_name(&metric.name, field), + &mut buf, + ); + buf + }) + .collect(); + Self { + order: sorted.into_iter().map(|(_, i)| i).collect(), + label_values, + names, + metric, + } + } +} + +impl Encoder for RecordsRemoteWriteEncoder { + fn encode(&self, rows: &Rows, range: Range) -> Vec { + let mut request = Vec::new(); + let mut series = Vec::new(); + for_each_record(&self.metric, rows, range, |timestamp_ms, labels, values| { + for (field, &value) in values.iter().enumerate() { + series.clear(); + for label in &self.order { + series.extend_from_slice(match label { + None => &self.names[field], + Some(l) => &self.label_values[*l][labels[*l] as usize], + }); + } + encode_sample(value, timestamp_ms, &mut series); + encode_bytes_field(1, &series, &mut request); + } + }); + compress(&request) } } @@ -94,8 +209,7 @@ impl RowBinaryEncoder { .map(|values| { let mut buf = Vec::new(); for s in values { - encode_varint(s.len() as u64, &mut buf); - buf.extend_from_slice(s.as_bytes()); + encode_string(s, &mut buf); } buf }) @@ -105,7 +219,8 @@ impl RowBinaryEncoder { } impl Encoder for RowBinaryEncoder { - fn encode(&self, rows: &[Row]) -> Vec { + fn encode(&self, rows: &Rows, range: Range) -> Vec { + let rows = &samples(rows)[range]; let mut buf = Vec::with_capacity(rows.len() * 32); for row in rows { buf.extend_from_slice(&row.timestamp_ms.to_le_bytes()); @@ -116,6 +231,96 @@ impl Encoder for RowBinaryEncoder { } } +/// ClickHouse `RowBinary` of a records metric for columns `ts DateTime64(3)`, +/// each label as `String` in description order, then each value field as +/// `Float64` in name order. +pub struct RecordsRowBinaryEncoder { + metric: Arc, + /// Each value of each label as a varint length plus bytes. + label_values: Vec>>, +} + +/// The column list `RecordsRowBinaryEncoder` writes for `metric`, in order. +pub fn record_columns(metric: &RecordMetric) -> String { + let labels = metric.labels.iter().map(|l| l.name.as_str()); + std::iter::once("ts") + .chain(labels) + .chain(metric.fields.iter().map(String::as_str)) + .collect::>() + .join(", ") +} + +impl RecordsRowBinaryEncoder { + pub fn new(metric: Arc) -> Self { + let label_values = metric + .label_values + .iter() + .map(|values| { + values + .iter() + .map(|value| { + let mut buf = Vec::new(); + encode_string(value, &mut buf); + buf + }) + .collect() + }) + .collect(); + Self { + metric, + label_values, + } + } +} + +impl Encoder for RecordsRowBinaryEncoder { + fn encode(&self, rows: &Rows, range: Range) -> Vec { + let mut buf = Vec::with_capacity(range.len() * 64); + for_each_record(&self.metric, rows, range, |timestamp_ms, labels, values| { + buf.extend_from_slice(×tamp_ms.to_le_bytes()); + for (label_values, &v) in self.label_values.iter().zip(labels) { + buf.extend_from_slice(&label_values[v as usize]); + } + for value in values { + buf.extend_from_slice(&value.to_le_bytes()); + } + }); + buf + } +} + +/// A `Label` message as `TimeSeries` field 1. +fn encode_label(name: &str, value: &str, buf: &mut Vec) { + let mut label = Vec::new(); + encode_bytes_field(1, name.as_bytes(), &mut label); + encode_bytes_field(2, value.as_bytes(), &mut label); + encode_bytes_field(1, &label, buf); +} + +/// A `Sample` message as `TimeSeries` field 2. +fn encode_sample(value: f64, timestamp_ms: i64, series: &mut Vec) { + // One-byte keys of fields 1 (fixed 64-bit) and 2 (varint). + let len = 1 + 8 + 1 + encoded_len_varint(timestamp_ms as u64); + encode_key(2, WireType::LengthDelimited, series); + encode_varint(len as u64, series); + encode_key(1, WireType::SixtyFourBit, series); + series.extend_from_slice(&value.to_le_bytes()); + encode_key(2, WireType::Varint, series); + encode_varint(timestamp_ms as u64, series); +} + +/// A ClickHouse `String`: varint length plus bytes. +fn encode_string(s: &str, buf: &mut Vec) { + encode_varint(s.len() as u64, buf); + buf.extend_from_slice(s.as_bytes()); +} + +fn compress(request: &[u8]) -> Vec { + snap::raw::Encoder::new() + .compress_vec(request) + .expect("snappy compression of an in-memory buffer cannot fail") +} + fn encode_bytes_field(tag: u32, bytes: &[u8], buf: &mut Vec) { encode_key(tag, WireType::LengthDelimited, buf); buf.reserve(encoded_len_varint(bytes.len() as u64) + bytes.len()); @@ -169,6 +374,7 @@ mod tests { name: name.to_string(), cardinality, child_of: child_of.map(str::to_string), + skew: None, }; let labels = vec![ label("zone", 2, None), @@ -188,9 +394,14 @@ mod tests { Workload { samples_per_sec: 1, metrics: vec![metric], + records: Vec::new(), } } + fn encode_all(encoder: &dyn Encoder, rows: &[Row]) -> Vec { + encoder.encode(&Rows::Samples(rows.to_vec()), 0..rows.len()) + } + fn decode_remote_write(body: &[u8]) -> WriteRequest { let raw = snap::raw::Decoder::new().decompress_vec(body).unwrap(); WriteRequest::decode(raw.as_slice()).unwrap() @@ -199,9 +410,11 @@ mod tests { #[test] fn remote_write_decodes_to_one_series_per_row() { let w = grouped(); - let rows = w.slot_rows(0, 2, 0, 1_700_000_000_000); - let request = - decode_remote_write(&RemoteWriteEncoder::new(&w.metrics[0], "data").encode(&rows)); + let rows = w.series_rows(0, 2, 0, 1_700_000_000_000); + let request = decode_remote_write(&encode_all( + &RemoteWriteEncoder::new(&w.metrics[0], "data"), + &rows, + )); assert_eq!(request.timeseries.len(), rows.len()); for (ts, row) in request.timeseries.iter().zip(&rows) { let values = &w.metrics[0].series[row.series]; @@ -232,9 +445,11 @@ mod tests { fn remote_write_sorts_labels_by_name() { let w = multi_label(); let metric = &w.metrics[0]; - let rows = w.slot_rows(0, 0, 0, 1_000); - let request = - decode_remote_write(&RemoteWriteEncoder::new(metric, &metric.name).encode(&rows)); + let rows = w.series_rows(0, 0, 0, 1_000); + let request = decode_remote_write(&encode_all( + &RemoteWriteEncoder::new(metric, &metric.name), + &rows, + )); assert_eq!(request.timeseries.len(), 12); for (ts, row) in request.timeseries.iter().zip(&rows) { let [zone, job, label_0, az] = metric.series[row.series].as_slice() else { @@ -289,8 +504,8 @@ mod tests { let w = grouped(); let metric = &w.metrics[0]; assert_eq!(row_binary_columns(metric), "ts, label_0, instance, value"); - let rows = w.slot_rows(0, 0, 0, 42_000); - let decoded = decode_row_binary(&RowBinaryEncoder::new(metric).encode(&rows), 2); + let rows = w.series_rows(0, 0, 0, 42_000); + let decoded = decode_row_binary(&encode_all(&RowBinaryEncoder::new(metric), &rows), 2); let expected: Vec<_> = rows .iter() .map(|r| (r.timestamp_ms, metric.series[r.series].clone(), r.value)) @@ -306,8 +521,8 @@ mod tests { row_binary_columns(metric), "ts, zone, job, label_0, Az, value" ); - let rows = w.slot_rows(0, 3, 0, 0); - let decoded = decode_row_binary(&RowBinaryEncoder::new(metric).encode(&rows), 4); + let rows = w.series_rows(0, 3, 0, 0); + let decoded = decode_row_binary(&encode_all(&RowBinaryEncoder::new(metric), &rows), 4); assert_eq!(decoded.len(), 12); assert_eq!( decoded[4], @@ -318,4 +533,153 @@ mod tests { ) ); } + + fn http() -> Arc { + let spec = crate::records::tests::http_spec(); + Arc::new(RecordMetric::from_table("http", &spec, Some(1_000), 42).unwrap()) + } + + /// Record `index` of `metric` as (timestamp, label values, values). + fn record(metric: &RecordMetric, index: u64, start_ms: i64) -> (i64, Vec, Vec) { + let mut labels = vec![0; metric.labels.len()]; + let mut values = vec![0.0; metric.fields.len()]; + metric.record(index, &mut labels, &mut values); + let names = labels + .iter() + .zip(&metric.label_values) + .map(|(&v, values)| values[v as usize].clone()) + .collect(); + (metric.timestamp_ms(index, start_ms), names, values) + } + + #[test] + fn remote_write_sends_a_sample_per_record_field() { + let metric = http(); + // Slot 3 of tick 1: records 1030..1040 at 1_000 + 1_000 + 40 ms. + let rows = Rows::Records(Records { + start_ms: 1_000, + indices: 1_030..1_040, + }); + let request = decode_remote_write( + &RecordsRemoteWriteEncoder::new(Arc::clone(&metric)).encode(&rows, 2..5), + ); + assert_eq!(request.timeseries.len(), 6); + for (i, pair) in request.timeseries.chunks(2).enumerate() { + let (timestamp, labels, values) = record(&metric, 1_032 + i as u64, 1_000); + assert_eq!(timestamp, 2_040); + // Value fields in name order: latency, then user_id. + for (ts, (field, value)) in pair + .iter() + .zip([("latency", values[0]), ("user_id", values[1])]) + { + let got: Vec<(&str, &str)> = ts + .labels + .iter() + .map(|l| (l.name.as_str(), l.value.as_str())) + .collect(); + let name = format!("http.{field}"); + assert_eq!( + got, + vec![ + ("__name__", name.as_str()), + ("endpoint", labels[2].as_str()), + ("region", labels[0].as_str()), + ("service", labels[1].as_str()), + ("status", labels[3].as_str()), + ] + ); + assert_eq!(ts.samples, vec![Sample { value, timestamp }]); + } + } + } + + /// Decodes RowBinary rows of `ts`, `num_labels` strings and `num_values` floats. + fn decode_record_rows( + buf: &[u8], + num_labels: usize, + num_values: usize, + ) -> Vec<(i64, Vec, Vec)> { + let mut cursor = buf; + let mut take = |n: usize| -> Vec { + let (head, rest) = cursor.split_at(n); + cursor = rest; + head.to_vec() + }; + let mut rows = Vec::new(); + let mut remaining = buf.len(); + while remaining > 0 { + let ts = i64::from_le_bytes(take(8).try_into().unwrap()); + let labels: Vec = (0..num_labels) + .map(|_| { + let len = take(1)[0] as usize; + String::from_utf8(take(len)).unwrap() + }) + .collect(); + let values: Vec = (0..num_values) + .map(|_| f64::from_le_bytes(take(8).try_into().unwrap())) + .collect(); + remaining -= 8 + 8 * num_values + labels.iter().map(|l| 1 + l.len()).sum::(); + rows.push((ts, labels, values)); + } + rows + } + + #[test] + fn row_binary_writes_a_row_per_record() { + let metric = http(); + assert_eq!( + record_columns(&metric), + "ts, region, service, endpoint, status, latency, user_id" + ); + let rows = Rows::Records(metric.tick(0, 5_000)); + assert_eq!(rows.len(), 1_000); + let body = RecordsRowBinaryEncoder::new(Arc::clone(&metric)).encode(&rows, 0..1_000); + let decoded = decode_record_rows(&body, 4, 2); + let expected: Vec<_> = (0..1_000).map(|i| record(&metric, i, 5_000)).collect(); + assert_eq!(decoded, expected); + assert_eq!((decoded[0].0, decoded[999].0), (5_010, 6_000)); + } + + #[test] + fn both_sinks_send_the_same_records() { + let metric = http(); + let rows = Rows::Records(metric.tick(2, 0)); + let range = 0..rows.len(); + let request = decode_remote_write( + &RecordsRemoteWriteEncoder::new(Arc::clone(&metric)).encode(&rows, range.clone()), + ); + let decoded = decode_record_rows( + &RecordsRowBinaryEncoder::new(Arc::clone(&metric)).encode(&rows, range), + 4, + 2, + ); + assert_eq!(request.timeseries.len(), 2 * decoded.len()); + for (pair, (ts, labels, values)) in request.timeseries.chunks(2).zip(&decoded) { + for (series, value) in pair.iter().zip(values) { + assert_eq!( + series.samples, + vec![Sample { + value: *value, + timestamp: *ts + }] + ); + let mut sent: Vec<&str> = series.labels[1..] + .iter() + .map(|l| l.value.as_str()) + .collect(); + let mut want: Vec<&str> = labels.iter().map(String::as_str).collect(); + sent.sort_unstable(); + want.sort_unstable(); + assert_eq!(sent, want); + } + } + } + + #[test] + #[should_panic(expected = "a series encoder was given records")] + fn series_encoders_reject_records() { + let w = grouped(); + let rows = Rows::Records(http().tick(0, 0)); + RowBinaryEncoder::new(&w.metrics[0]).encode(&rows, 0..1); + } } diff --git a/asap-tools/data-sources/paced-feeder/src/main.rs b/asap-tools/data-sources/paced-feeder/src/main.rs index f7e1efff..c466cf15 100644 --- a/asap-tools/data-sources/paced-feeder/src/main.rs +++ b/asap-tools/data-sources/paced-feeder/src/main.rs @@ -7,17 +7,21 @@ //! is written to `--stats-out` so a run can be checked for keeping up. mod encode; +mod records; mod table; mod workload; use anyhow::{bail, Context, Result}; use clap::{Parser, ValueEnum}; -use encode::{row_binary_columns, Encoder, RemoteWriteEncoder, RowBinaryEncoder}; +use encode::{ + record_columns, row_binary_columns, Encoder, RecordsRemoteWriteEncoder, + RecordsRowBinaryEncoder, RemoteWriteEncoder, RowBinaryEncoder, +}; use serde::Serialize; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::sync::Semaphore; -use workload::{Row, Workload}; +use workload::{Rows, Workload}; #[derive(Debug, Clone, Copy, ValueEnum, Serialize)] #[serde(rename_all = "kebab-case")] @@ -65,6 +69,10 @@ struct Args { /// the table's `samples_per_sec_per_series` with `--workload-table`. #[arg(long)] samples_per_sec: Option, + /// Records per second of each `records` metric of `--workload-table`, + /// overriding the table's `records_per_sec`, e.g. to scale a run down. + #[arg(long, requires = "workload_table")] + records_per_sec: Option, #[arg(long, default_value_t = 1.5)] pareto_shape: f64, #[arg(long, default_value_t = 1.0)] @@ -72,8 +80,9 @@ struct Args { #[arg(long, default_value_t = 42)] seed: u64, /// Print, for every subset of each metric's labels, the expected and the - /// generated number of distinct label-value tuples, then exit without - /// sending; non-zero if any differ. + /// generated number of distinct label-value tuples, and for records + /// metrics the expected and generated shares of each label's heaviest + /// values, then exit without sending; non-zero if any check fails. #[arg(long)] verify_facts: bool, @@ -87,7 +96,8 @@ struct Args { /// Rows per HTTP request. Defaults to 50,000 for remote write, which keeps /// a compressed request under axum's 2 MB body limit in ASAPQuery's ingest, - /// and 100,000 for ClickHouse, where each INSERT creates a part. + /// and 100,000 for ClickHouse, where each INSERT creates a part. For + /// remote write it counts samples: a record is one per value field. #[arg(long)] batch_rows: Option, /// Concurrent requests within one tick. @@ -119,7 +129,8 @@ struct RunStats { pareto_scale: Option, workload_table: Option, metrics: Vec, - samples_per_sec: u32, + /// Unset without gauge/counter metrics. + samples_per_sec: Option, seed: u64, start_ms: i64, duration_secs: u64, @@ -137,34 +148,57 @@ struct RunStats { ticks: Vec, } -/// Where one metric's rows go: the request to send them with and their encoder. +/// Where one metric's rows go: the request to send them with, their encoder +/// and the rows per request. struct Target { request: reqwest::RequestBuilder, encoder: Box, + batch_rows: usize, } struct Sink { client: reqwest::Client, - /// One per metric of the workload, in order. + /// One per metric of the workload, in [`Workload::names`] order. targets: Vec, } impl Sink { - fn new(args: &Args, sink: SinkKind, url: &str, workload: &Workload) -> Result { + fn new( + args: &Args, + sink: SinkKind, + url: &str, + workload: &Workload, + batch_rows: usize, + ) -> Result { let client = reqwest::Client::builder() .pool_max_idle_per_host(args.concurrency) .build()?; - let targets = workload + let remote_write = || { + client + .post(url) + .header("Content-Encoding", "snappy") + .header("Content-Type", "application/x-protobuf") + .header("X-Prometheus-Remote-Write-Version", "0.1.0") + }; + let clickhouse = |table: &str, columns: String| { + let query = format!("INSERT INTO {table} ({columns}) FORMAT RowBinary"); + let mut request = client.post(url).query(&[("query", query)]); + if let Some(user) = &args.clickhouse_user { + request = request.header("X-ClickHouse-User", user); + } + if let Some(password) = &args.clickhouse_password { + request = request.header("X-ClickHouse-Key", password); + } + request + }; + let mut targets: Vec = workload .metrics .iter() .map(|metric| match sink { SinkKind::RemoteWrite => Target { - request: client - .post(url) - .header("Content-Encoding", "snappy") - .header("Content-Type", "application/x-protobuf") - .header("X-Prometheus-Remote-Write-Version", "0.1.0"), + request: remote_write(), encoder: Box::new(RemoteWriteEncoder::new(metric, &metric.name)), + batch_rows, }, SinkKind::Clickhouse => { let table = if args.workload_table.is_some() { @@ -172,30 +206,38 @@ impl Sink { } else { &args.table }; - let query = format!( - "INSERT INTO {table} ({}) FORMAT RowBinary", - row_binary_columns(metric) - ); - let mut request = client.post(url).query(&[("query", query)]); - if let Some(user) = &args.clickhouse_user { - request = request.header("X-ClickHouse-User", user); - } - if let Some(password) = &args.clickhouse_password { - request = request.header("X-ClickHouse-Key", password); - } Target { - request, + request: clickhouse(table, row_binary_columns(metric)), encoder: Box::new(RowBinaryEncoder::new(metric)), + batch_rows, } } }) .collect(); + targets.extend(workload.records.iter().map(|metric| match sink { + // Remote write's batch counts samples, one per value field. + SinkKind::RemoteWrite => Target { + request: remote_write(), + encoder: Box::new(RecordsRemoteWriteEncoder::new(Arc::clone(metric))), + batch_rows: (batch_rows / metric.fields.len()).max(1), + }, + SinkKind::Clickhouse => Target { + request: clickhouse(&metric.name, record_columns(metric)), + encoder: Box::new(RecordsRowBinaryEncoder::new(Arc::clone(metric))), + batch_rows, + }, + })); Ok(Self { client, targets }) } - async fn send(&self, metric: usize, rows: &[Row]) -> Result { + async fn send( + &self, + metric: usize, + rows: &Rows, + range: std::ops::Range, + ) -> Result { let target = &self.targets[metric]; - let body = target.encoder.encode(rows); + let body = target.encoder.encode(rows, range); let bytes = body.len(); let request = target .request @@ -239,6 +281,7 @@ fn build_workload(args: &Args) -> Result { path, args.metric.as_deref(), args.samples_per_sec, + args.records_per_sec, args.seed, ); } @@ -264,22 +307,67 @@ fn build_workload(args: &Args) -> Result { } /// Prints the facts check of every metric; true if every fact holds. -fn verify_facts(workload: &Workload) -> bool { +fn verify_facts(workload: &Workload) -> Result { let mut ok = true; for metric in &workload.metrics { println!("{}: {} series", metric.name, metric.series.len()); - println!(" {:<40} {:>12} {:>12}", "labels", "expected", "generated"); - for fact in table::facts(&metric.labels, &metric.series) { - let holds = fact.expected == fact.actual; + ok &= print_label_facts(&metric.labels, &metric.series); + } + for metric in &workload.records { + // Every combination of label values a record can have. + let domain = table::series_labels(&metric.labels)?; + println!( + "{}: {} records/s, {} label combinations", + metric.name, + metric.records_per_sec, + domain.len() + ); + ok &= print_label_facts(&metric.labels, &domain); + let report = metric.share_checks(records::SHARE_SAMPLE_RECORDS); + println!( + " shares of {} records (tolerance {} standard errors)", + report.records, + records::SHARE_TOLERANCE_SIGMAS + ); + println!( + " {:<12} {:<28} {:>10} {:>10} {:>10}", + "label", "value", "expected", "generated", "tolerance" + ); + for check in &report.checks { + let holds = check.holds(); ok &= holds; println!( - " {:<40} {:>12} {:>12}{}", - fact.labels.join(","), - fact.expected, - fact.actual, + " {:<12} {:<28} {:>10.5} {:>10.5} {:>10.5}{}", + check.label, + check.value, + check.expected, + check.actual, + check.tolerance, if holds { "" } else { " MISMATCH" } ); } + if let Some((seen, keys)) = report.burst_keys { + println!(" burst keys: {seen} distinct of {keys}"); + } + } + Ok(ok) +} + +/// Prints the distinct label-value tuples of every label subset of `series`; +/// true if each matches the description. +fn print_label_facts(labels: &[workload::LabelSpec], series: &[Vec]) -> bool { + let mut ok = true; + println!(" {:<40} {:>12} {:>12}", "labels", "expected", "generated"); + for fact in table::facts(labels, series) { + let holds = fact.expected == fact.actual; + ok &= holds; + println!( + " {:<40} {:>12} {:>12}{}", + fact.labels.join(","), + fact.expected, + fact.actual, + if holds { "" } else { " MISMATCH" } + ); } ok } @@ -288,17 +376,19 @@ fn verify_facts(workload: &Workload) -> bool { async fn main() -> Result<()> { let args = Args::parse(); let workload = build_workload(&args)?; - if workload.samples_per_sec == 0 { - bail!("samples per second must be positive"); - } - // Timestamps are in ms, so more samples per second would repeat a timestamp - // within a series, which remote write drops and ClickHouse keeps. - if workload.samples_per_sec > 1_000 { - bail!("samples per second must be at most 1000"); + if !workload.metrics.is_empty() { + if workload.samples_per_sec == 0 { + bail!("samples per second must be positive"); + } + // Timestamps are in ms, so more samples per second would repeat a + // timestamp within a series, which remote write drops and ClickHouse keeps. + if workload.samples_per_sec > 1_000 { + bail!("samples per second must be at most 1000"); + } } if args.verify_facts { - if !verify_facts(&workload) { - bail!("the generated series do not match the description"); + if !verify_facts(&workload)? { + bail!("the generated data does not match the description"); } return Ok(()); } @@ -325,16 +415,23 @@ async fn main() -> Result<()> { bail!("--batch-rows and --concurrency must be positive"); } - let sink = Arc::new(Sink::new(&args, sink_kind, url, &workload)?); + let sink = Arc::new(Sink::new(&args, sink_kind, url, &workload, batch_rows)?); let permits = Arc::new(Semaphore::new(args.concurrency)); let start_ms = args.start_ms.unwrap_or_else(next_whole_second_ms); - let metric_names: Vec = workload.metrics.iter().map(|m| m.name.clone()).collect(); + let metric_names = workload.names(); + let series = if workload.metrics.is_empty() { + String::new() + } else { + format!( + " ({} series x {}/s)", + workload.num_series(), + workload.samples_per_sec + ) + }; eprintln!( - "paced_feeder: {sink_kind:?} -> {url}, {} ({} series) x {}/s = {} rows/s for {duration_secs}s, start_ms={start_ms}", + "paced_feeder: {sink_kind:?} -> {url}, {}{series} = {} rows/s for {duration_secs}s, start_ms={start_ms}", metric_names.join(", "), - workload.num_series(), - workload.samples_per_sec, workload.rows_per_tick(), ); @@ -351,22 +448,19 @@ async fn main() -> Result<()> { let (mut rows, mut bytes) = (0, 0); // A tick's requests go out concurrently, so a group's samples within // the tick may arrive out of order. That is safe for ASAPQuery: the - // tick lies in one window (see `slot_rows`) and is under one second + // tick lies in one window (see `tick_rows`) and is under one second // behind the watermark, within the allowed lateness. let mut tasks = tokio::task::JoinSet::new(); - for metric in 0..workload.metrics.len() { - let metric_rows: Arc> = Arc::new( - (0..workload.samples_per_sec) - .flat_map(|slot| workload.slot_rows(metric, tick, slot, start_ms)) - .collect(), - ); + for metric in 0..metric_names.len() { + let metric_rows = Arc::new(workload.tick_rows(metric, tick, start_ms)); + let batch_rows = sink.targets[metric].batch_rows; for begin in (0..metric_rows.len()).step_by(batch_rows) { let end = (begin + batch_rows).min(metric_rows.len()); let (metric_rows, sink) = (Arc::clone(&metric_rows), Arc::clone(&sink)); let permit = Arc::clone(&permits).acquire_owned().await?; tasks.spawn(async move { let _permit = permit; - let sent = sink.send(metric, &metric_rows[begin..end]).await; + let sent = sink.send(metric, &metric_rows, begin..end).await; sent.map(|bytes| (end - begin, bytes)) }); } @@ -424,7 +518,7 @@ async fn main() -> Result<()> { pareto_scale: grouped.then_some(args.pareto_scale), workload_table: args.workload_table.clone(), metrics: metric_names, - samples_per_sec: workload.samples_per_sec, + samples_per_sec: (!workload.metrics.is_empty()).then_some(workload.samples_per_sec), seed: args.seed, start_ms, duration_secs, diff --git a/asap-tools/data-sources/paced-feeder/src/records.rs b/asap-tools/data-sources/paced-feeder/src/records.rs new file mode 100644 index 00000000..86148c10 --- /dev/null +++ b/asap-tools/data-sources/paced-feeder/src/records.rs @@ -0,0 +1,880 @@ +//! `records` metrics of a workload table: a stream of records, each with a +//! value of every label and of every value field, e.g. sketch-bench's `http` +//! (`region`, `service`, `endpoint`, `status`; `latency`, `user_id`). +//! +//! Record `n` depends only on `(metric seed, n)`: every draw is a stateless +//! hash of the seed, the record index and the draw's stream (label `i` is +//! stream `i`, value field `j` is stream `labels + j`, the burst coin is +//! stream `labels + fields`). The data matches sketch-bench's +//! `label_shares()`: +//! +//! - **Labels.** Each label is drawn independently (a child given its parent) +//! from Zipf(`skew`) over its `cardinality` values (a child's: under its +//! parent), the value of index `i` having weight `(i+1)^-skew`, so `0` is the +//! heaviest. Values are named as for gauges: `"3"`, or a child's `"3.17"`. +//! - **Keys** (`{"zipf": θ, "keys": K}`): the key of Zipf(θ) rank `r` over `K` +//! keys is `r`. +//! - **Burst** (`"burst": {"label", "value", "share", "keys"}` on a keys field): +//! with probability `share` a record is a burst record: its `label` is +//! `value` (a child index for a child label) and its key is uniform over the +//! `keys` burst keys `K, K+1, …`, disjoint from the background keys. Other +//! records draw both as above, so `label`'s shares are Zipf mixed with the +//! burst: `(1 - share)·p_i`, plus `share` at `value`. +//! - **Pareto** (`{"pareto": a}`): `PARETO_SCALE · U^(-1/a)`, times, with +//! `"scaled_by": L`, the fixed factor of the record's value of `L` +//! (log-uniform in `SCALE_RANGE`, from a hash of the value and the feeder's +//! seed), times, with `"anomaly": {"label": L, "factor": f}`, `f` when the +//! record's value of `L` is its first one (`0`, or `0.0` for a child: the +//! heaviest). Scaling keeps the tail index `a`. + +use crate::table::{fnv1a, is_identifier, label_parents}; +use crate::workload::{splitmix64, unit_uniform, LabelSpec}; +use anyhow::{bail, ensure, Context, Result}; +use serde::Deserialize; +use serde_json::Value; +use std::collections::{BTreeMap, HashSet}; +use std::ops::Range; + +/// Records are timestamped to 10 ms slots and spread evenly across them. +pub const SLOTS_PER_SEC: u32 = 100; +const SLOT_MS: i64 = 1_000 / SLOTS_PER_SEC as i64; +/// Scale of `pareto` values, as in sketch-bench's +/// `configs/datagen/hydra_http_latency.yaml` (`scale: 1000.0`). +pub const PARETO_SCALE: f64 = 1_000.0; +/// `[lo, hi]` of the `scaled_by` factors, as in the same spec's `scale_range`. +pub const SCALE_RANGE: [f64; 2] = [1.0, 10.0]; +/// Keys are sent as `Float64`, which holds integers exactly up to 2^53. +const MAX_KEY: u64 = 1 << 53; +/// The facts check counts shares over this many records. +pub const SHARE_SAMPLE_RECORDS: u64 = 1_000_000; +/// A share's check fails if it is off by more than this many standard errors +/// of a share estimated from the sample, `sqrt(p(1-p)/n)`. +pub const SHARE_TOLERANCE_SIGMAS: f64 = 5.0; +/// The facts check shows each label's heaviest values. +const TOP_SHARES: usize = 3; + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RecordsSpec { + #[serde(rename = "type")] + _kind: String, + labels: Vec, + records_per_sec: u64, + values: BTreeMap, +} + +/// Either `zipf` and `keys` (with an optional `burst`), or `pareto` (with +/// optional `scaled_by` and `anomaly`). +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct FieldSpec { + zipf: Option, + keys: Option, + burst: Option, + pareto: Option, + scaled_by: Option, + anomaly: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct BurstSpec { + label: String, + value: u64, + share: f64, + keys: u64, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct AnomalySpec { + label: String, + factor: f64, +} + +/// Zipf over `n` values, value `i` with weight `(i+1)^-theta`, drawn by +/// inverse transform: the first value whose CDF reaches `u`. A guide table +/// narrows the binary search to the values whose CDF crosses `u`'s bucket, so +/// a draw over 1e6 keys touches a few cache lines instead of 20. +#[derive(Debug, Clone, PartialEq)] +pub struct Zipf { + cdf: Vec, + /// `guide[j]`: the first value whose CDF reaches `j / n`, for `j` in `0..=n`. + guide: Vec, +} + +/// Zipf(`theta`) shares of `n` values, the value of index `i` with weight +/// `(i+1)^-theta`. +pub fn zipf_shares(n: u64, theta: f64) -> Vec { + let weights: Vec = (0..n).map(|i| ((i + 1) as f64).powf(-theta)).collect(); + let total: f64 = weights.iter().sum(); + weights.into_iter().map(|w| w / total).collect() +} + +impl Zipf { + pub fn new(n: u64, theta: f64) -> Self { + let mut cdf: Vec = zipf_shares(n, theta) + .into_iter() + .scan(0.0, |sum, p| { + *sum += p; + Some(*sum) + }) + .collect(); + // Rounding can leave the sum just under 1; any `u` must find a value. + *cdf.last_mut().expect("a Zipf has values") = 1.0; + let buckets = cdf.len(); + let mut guide = Vec::with_capacity(buckets + 1); + let mut i = 0; + for j in 0..=buckets { + let bound = j as f64 / buckets as f64; + while cdf[i] < bound { + i += 1; + } + guide.push(i as u32); + } + Self { cdf, guide } + } + + /// The value index for `u` in (0, 1]. + pub fn draw(&self, u: f64) -> u64 { + // `u` is in bucket `j`: (j/n, (j+1)/n], so its value lies in + // `guide[j]..=guide[j+1]`; one bucket more on each side absorbs + // rounding in `u·n`. + let buckets = self.cdf.len(); + let j = ((u * buckets as f64).ceil() as usize).clamp(1, buckets) - 1; + let lo = self.guide[j.saturating_sub(1)] as usize; + let hi = self.guide[(j + 2).min(buckets)] as usize; + (lo + self.cdf[lo..=hi].partition_point(|&c| c < u)) as u64 + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Burst { + /// The label and value (a child index for a child label) of burst records. + pub label: usize, + pub value: u64, + pub share: f64, + /// The keys field and its number of burst keys, numbered after its keys. + pub field: usize, + pub keys: u64, +} + +#[derive(Debug, Clone, PartialEq)] +enum Field { + Keys { + zipf: Zipf, + keys: u64, + }, + Pareto { + shape: f64, + /// The `scaled_by` label and each of its values' factor. + scaled_by: Option<(usize, Vec)>, + /// The anomaly label and factor; the anomalous value is the label's first. + anomaly: Option<(usize, f64)>, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RecordMetric { + pub name: String, + pub labels: Vec, + /// Every value of each label, a child's across all parents: value `v` of a + /// record is `label_values[label][v]`. + pub label_values: Vec>, + parents: Vec>, + label_zipfs: Vec, + /// Value field names, in name order. + pub fields: Vec, + field_kinds: Vec, + pub burst: Option, + pub records_per_sec: u64, + seed: u64, +} + +/// Records `indices` of a metric, with event time from `start_ms`. Records +/// are generated as they are encoded, so each request's are generated by its +/// own task. +#[derive(Debug, Clone, PartialEq)] +pub struct Records { + pub start_ms: i64, + pub indices: Range, +} + +impl Records { + pub fn len(&self) -> usize { + (self.indices.end - self.indices.start) as usize + } +} + +impl RecordMetric { + /// A `records` metric from its table description. `records_per_sec` + /// overrides the table's rate; `seed` is the feeder's. + pub fn from_table( + name: &str, + spec: &Value, + records_per_sec: Option, + seed: u64, + ) -> Result { + let spec: RecordsSpec = serde_json::from_value(spec.clone())?; + let labels = spec.labels; + let parents = label_parents(&labels)?; + let label_index = |name: &str| -> Result { + labels + .iter() + .position(|l| l.name == name) + .with_context(|| format!("no label `{name}`")) + }; + let mut label_zipfs = Vec::new(); + for label in &labels { + let skew = label + .skew + .with_context(|| format!("label `{}` needs a `skew`", label.name))?; + ensure!(skew >= 0.0, "label `{}`: skew must be ≥ 0", label.name); + ensure!( + label.cardinality <= u64::from(u32::MAX), + "label `{}`: too many values", + label.name + ); + label_zipfs.push(Zipf::new(label.cardinality, skew)); + } + let label_values = label_values(&labels, &parents)?; + + ensure!(!spec.values.is_empty(), "a records metric needs values"); + let mut burst = None; + let mut fields = Vec::new(); + let mut field_kinds = Vec::new(); + for (field, value) in spec.values { + ensure!( + is_identifier(&field) && field != "ts" && label_index(&field).is_err(), + "value `{field}` must match [A-Za-z_][A-Za-z0-9_]* and not be `ts` or a label" + ); + let kind = match value { + FieldSpec { + zipf: Some(theta), + keys: Some(keys), + burst: field_burst, + pareto: None, + scaled_by: None, + anomaly: None, + } => { + ensure!(theta >= 0.0, "value `{field}`: zipf must be ≥ 0"); + ensure!( + keys > 0 && keys <= MAX_KEY, + "value `{field}`: keys must be in [1, 2^53]" + ); + if let Some(b) = field_burst { + ensure!(burst.is_none(), "at most one value may have a burst"); + let label = label_index(&b.label) + .with_context(|| format!("value `{field}`: burst"))?; + ensure!( + b.value < labels[label].cardinality, + "value `{field}`: the burst value is not a value of `{}`", + b.label + ); + ensure!( + b.share > 0.0 && b.share < 1.0, + "value `{field}`: the burst share must be in (0, 1)" + ); + ensure!( + b.keys > 0 && keys.saturating_add(b.keys) <= MAX_KEY, + "value `{field}`: burst keys must be positive, and all keys ≤ 2^53" + ); + burst = Some(Burst { + label, + value: b.value, + share: b.share, + field: fields.len(), + keys: b.keys, + }); + } + Field::Keys { + zipf: Zipf::new(keys, theta), + keys, + } + } + FieldSpec { + zipf: None, + keys: None, + burst: None, + pareto: Some(shape), + scaled_by, + anomaly, + } => { + ensure!(shape > 0.0, "value `{field}`: pareto must be positive"); + let scaled_by = match scaled_by { + None => None, + Some(by) => { + let label = label_index(&by) + .with_context(|| format!("value `{field}`: scaled_by"))?; + let factors = label_values[label] + .iter() + .map(|v| scale_factor(v, seed)) + .collect(); + Some((label, factors)) + } + }; + let anomaly = match anomaly { + None => None, + Some(a) => { + ensure!( + a.factor > 0.0, + "value `{field}`: the anomaly factor must be positive" + ); + let label = label_index(&a.label) + .with_context(|| format!("value `{field}`: anomaly"))?; + Some((label, a.factor)) + } + }; + Field::Pareto { + shape, + scaled_by, + anomaly, + } + } + _ => bail!( + "value `{field}` must be either `zipf` and `keys` (and `burst`), \ + or `pareto` (and `scaled_by`, `anomaly`)" + ), + }; + fields.push(field); + field_kinds.push(kind); + } + let records_per_sec = records_per_sec.unwrap_or(spec.records_per_sec); + ensure!(records_per_sec > 0, "records per second must be positive"); + Ok(Self { + name: name.to_string(), + labels, + label_values, + parents, + label_zipfs, + fields, + field_kinds, + burst, + records_per_sec, + seed: splitmix64(seed ^ fnv1a(name)), + }) + } + + /// Record `index`: writes each label's value index to `labels` and each + /// field's value to `values`. + pub fn record(&self, index: u64, labels: &mut [u32], values: &mut [f64]) { + let num_labels = self.labels.len() as u64; + let u = |stream: u64| unit_uniform(self.seed, stream, index); + let burst = self.burst.as_ref().filter(|b| { + let stream = num_labels + self.fields.len() as u64; + u(stream) <= b.share + }); + for (i, zipf) in self.label_zipfs.iter().enumerate() { + let child = match burst { + Some(b) if b.label == i => b.value, + _ => zipf.draw(u(i as u64)), + }; + labels[i] = match self.parents[i] { + None => child, + Some(p) => u64::from(labels[p]) * self.labels[i].cardinality + child, + } as u32; + } + for (j, kind) in self.field_kinds.iter().enumerate() { + let u = u(num_labels + j as u64); + values[j] = match kind { + Field::Keys { zipf, keys } => match burst { + // `u` in (0, 1] maps to 0..burst keys. + Some(b) if b.field == j => { + (keys + (u * b.keys as f64).ceil() as u64 - 1) as f64 + } + _ => zipf.draw(u) as f64, + }, + Field::Pareto { + shape, + scaled_by, + anomaly, + } => { + let mut value = PARETO_SCALE * u.powf(-1.0 / shape); + if let Some((label, factors)) = scaled_by { + value *= factors[labels[*label] as usize]; + } + if let Some((label, factor)) = anomaly { + if labels[*label] == 0 { + value *= factor; + } + } + value + } + }; + } + } + + /// The records of one-second tick `tick`, numbered from + /// `tick · records_per_sec`, with event time starting at `start_ms`. + pub fn tick(&self, tick: u64, start_ms: i64) -> Records { + let first = tick * self.records_per_sec; + Records { + start_ms, + indices: first..first + self.records_per_sec, + } + } + + /// The event time of record `index`: the end of its 10 ms slot, so a + /// tick starting at `t` has its records in `(t, t + 1 s]`. A second's + /// records are spread evenly over its slots, slot `s` holding offsets + /// `[s · r / 100, (s + 1) · r / 100)`. + pub fn timestamp_ms(&self, index: u64, start_ms: i64) -> i64 { + let (tick, offset) = (index / self.records_per_sec, index % self.records_per_sec); + let slot = (u64::from(SLOTS_PER_SEC) * (offset + 1) - 1) / self.records_per_sec; + start_ms + tick as i64 * 1_000 + (slot as i64 + 1) * SLOT_MS + } + + /// Each value's share of the records (a child's: given its parent), as + /// sketch-bench's `label_shares()`: Zipf(`skew`), mixed with the burst. + pub fn label_shares(&self, label: usize) -> Vec { + let spec = &self.labels[label]; + let mut shares = zipf_shares(spec.cardinality, spec.skew.unwrap_or_default()); + if let Some(b) = self.burst.as_ref().filter(|b| b.label == label) { + for p in &mut shares { + *p *= 1.0 - b.share; + } + shares[b.value as usize] += b.share; + } + shares + } + + /// The share checks of the facts check over records `0..n`: each label's + /// heaviest values (a child's index pooled over parents, so its share given + /// the parent), the burst value and the share of burst records. + pub fn share_checks(&self, n: u64) -> ShareReport { + let (num_labels, num_fields) = (self.labels.len(), self.fields.len()); + let mut counts: Vec> = self + .labels + .iter() + .map(|l| vec![0; l.cardinality as usize]) + .collect(); + let (mut burst_records, mut burst_keys) = (0u64, HashSet::new()); + let (mut labels, mut values) = (vec![0u32; num_labels], vec![0.0; num_fields]); + for index in 0..n { + self.record(index, &mut labels, &mut values); + for (i, &v) in labels.iter().enumerate() { + counts[i][(u64::from(v) % self.labels[i].cardinality) as usize] += 1; + } + if let Some(b) = &self.burst { + let Field::Keys { keys, .. } = self.field_kinds[b.field] else { + unreachable!("a burst is on a keys field"); + }; + let key = values[b.field] as u64; + if key >= keys { + burst_records += 1; + burst_keys.insert(key); + } + } + } + let check = |label: String, value: String, expected: f64, count: u64| ShareCheck { + label, + value, + expected, + actual: count as f64 / n as f64, + tolerance: SHARE_TOLERANCE_SIGMAS * (expected * (1.0 - expected) / n as f64).sqrt(), + }; + let mut checks = Vec::new(); + for (i, label) in self.labels.iter().enumerate() { + let shares = self.label_shares(i); + let mut shown: Vec = (0..shares.len()).collect(); + shown.sort_by(|&a, &b| shares[b].total_cmp(&shares[a]).then(a.cmp(&b))); + shown.truncate(TOP_SHARES); + if let Some(b) = self.burst.as_ref().filter(|b| b.label == i) { + if !shown.contains(&(b.value as usize)) { + shown.push(b.value as usize); + } + } + for v in shown { + let value = match &label.child_of { + None => v.to_string(), + Some(parent) => format!("*.{v} (given {parent})"), + }; + checks.push(check(label.name.clone(), value, shares[v], counts[i][v])); + } + } + let burst = self.burst.as_ref().map(|b| { + checks.push(check( + "burst".to_string(), + "records".to_string(), + b.share, + burst_records, + )); + (burst_keys.len() as u64, b.keys) + }); + ShareReport { + records: n, + checks, + burst_keys: burst, + } + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ShareCheck { + pub label: String, + pub value: String, + pub expected: f64, + pub actual: f64, + pub tolerance: f64, +} + +impl ShareCheck { + pub fn holds(&self) -> bool { + (self.actual - self.expected).abs() <= self.tolerance + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ShareReport { + pub records: u64, + pub checks: Vec, + /// Distinct burst keys among the records, and the burst's keys. + pub burst_keys: Option<(u64, u64)>, +} + +/// Every value of each label: a root's `0`, `1`, …; a child's, for each parent +/// value in order, `parent.0`, `parent.1`, … +fn label_values(labels: &[LabelSpec], parents: &[Option]) -> Result>> { + let mut values: Vec> = Vec::with_capacity(labels.len()); + for (label, parent) in labels.iter().zip(parents) { + let children = 0..label.cardinality; + let label_values: Vec = match parent { + None => children.map(|i| i.to_string()).collect(), + Some(p) => values[*p] + .iter() + .flat_map(|parent| children.clone().map(move |i| format!("{parent}.{i}"))) + .collect(), + }; + ensure!( + label_values.len() <= u32::MAX as usize, + "label `{}` has too many values", + label.name + ); + values.push(label_values); + } + Ok(values) +} + +/// The fixed `scaled_by` factor of label value `value`, as aqpbm-datagen's +/// `scale_factor`: a SplitMix64 hash of the value's FNV-1a and the seed, as +/// `u` in [0, 1), mapped to `lo·(hi/lo)^u`. +pub fn scale_factor(value: &str, seed: u64) -> f64 { + let [lo, hi] = SCALE_RANGE; + let u = (splitmix64(fnv1a(value) ^ seed) >> 11) as f64 / (1u64 << 53) as f64; + lo * (hi / lo).powf(u) +} + +#[cfg(test)] +pub mod tests { + use super::*; + use serde_json::json; + + /// sketch-bench's `http` schema, as an e2e table describes it. + pub fn http_spec() -> Value { + json!({"type": "records", "records_per_sec": 2_000_000, + "labels": [{"name": "region", "cardinality": 4, "skew": 0.5}, + {"name": "service", "cardinality": 25, "skew": 1.1}, + {"name": "endpoint", "cardinality": 25, "child_of": "service", "skew": 1.1}, + {"name": "status", "cardinality": 4, "skew": 2.0}], + "values": {"latency": {"pareto": 2.0, "scaled_by": "service", + "anomaly": {"label": "endpoint", "factor": 10}}, + "user_id": {"zipf": 0.8, "keys": 1_000_000}}}) + } + + /// sketch-bench's `flows` schema, as an e2e table describes it. + pub fn flows_spec() -> Value { + json!({"type": "records", "records_per_sec": 4_000_000, + "labels": [{"name": "dst_subnet", "cardinality": 1000, "skew": 1.1}, + {"name": "dst_port", "cardinality": 1000, "skew": 1.2}, + {"name": "proto", "cardinality": 3, "skew": 0.0}], + "values": {"src_ip": {"zipf": 0.0, "keys": 1_000_000, + "burst": {"label": "dst_subnet", "value": 999, + "share": 0.05, "keys": 10_000}}}}) + } + + fn metric(name: &str, spec: &Value) -> RecordMetric { + RecordMetric::from_table(name, spec, None, 42).unwrap() + } + + /// Records `0..n` as (label value indices, values). + fn records(m: &RecordMetric, n: u64) -> Vec<(Vec, Vec)> { + (0..n) + .map(|index| { + let mut labels = vec![0; m.labels.len()]; + let mut values = vec![0.0; m.fields.len()]; + m.record(index, &mut labels, &mut values); + (labels, values) + }) + .collect() + } + + fn within(actual: f64, expected: f64, n: usize) -> bool { + let sigma = (expected * (1.0 - expected) / n as f64).sqrt(); + (actual - expected).abs() <= SHARE_TOLERANCE_SIGMAS * sigma + } + + #[test] + fn zipf_guide_table_draws_as_a_plain_cdf_search() { + for (n, theta) in [(1, 0.0), (4, 2.0), (25, 1.1), (1_000, 0.0), (100_000, 0.8)] { + let zipf = Zipf::new(n, theta); + let plain = |u: f64| zipf.cdf.partition_point(|&c| c < u) as u64; + // Hashed draws, every bucket boundary and the ends of (0, 1]. + let us = (0..20_000) + .map(|i| unit_uniform(7, 0, i)) + .chain((1..=n.min(1_000)).map(|j| j as f64 / n as f64)) + .chain([f64::MIN_POSITIVE, 1e-12, 1.0 - 1e-16, 1.0]); + for u in us { + assert_eq!(zipf.draw(u), plain(u), "n={n} theta={theta} u={u}"); + } + } + } + + #[test] + fn label_shares_match_sketch_bench() { + // `label_shares()` of scripts/export_autosketch_eval_table.py. + let http = metric("http", &http_spec()); + let round = |p: f64| (p * 1e5).round() / 1e5; + let top3 = |m: &RecordMetric, l| -> Vec { + m.label_shares(l)[..3].iter().map(|&p| round(p)).collect() + }; + assert_eq!(top3(&http, 0), [0.35914, 0.25395, 0.20735]); + assert_eq!(top3(&http, 1), [0.29841, 0.13922, 0.08912]); + assert_eq!(top3(&http, 2), [0.29841, 0.13922, 0.08912]); + assert_eq!(top3(&http, 3), [0.70244, 0.17561, 0.07805]); + let flows = metric("flows", &flows_spec()); + assert_eq!(top3(&flows, 0), [0.17047, 0.07953, 0.05091]); + assert_eq!(round(flows.label_shares(0)[999]), 0.05009); + assert!((flows.label_shares(0).iter().sum::() - 1.0).abs() < 1e-9); + } + + #[test] + fn labels_follow_their_shares_and_children_their_parent() { + let http = metric("http", &http_spec()); + let n = 200_000; + let records = records(&http, n); + for (l, label) in http.labels.iter().enumerate() { + let shares = http.label_shares(l); + for (v, &expected) in shares.iter().enumerate().take(5) { + let count = records + .iter() + .filter(|(labels, _)| labels[l] as u64 % label.cardinality == v as u64) + .count(); + let actual = count as f64 / n as f64; + assert!( + within(actual, expected, n as usize), + "{}={v}: {actual} vs {expected}", + label.name + ); + } + } + // A child value names its parent, and given a parent the children + // follow their own Zipf shares. + let (service, endpoint) = (1, 2); + for (labels, _) in &records { + let name = &http.label_values[endpoint][labels[endpoint] as usize]; + let parent = &http.label_values[service][labels[service] as usize]; + assert!( + name.starts_with(&format!("{parent}.")), + "{name} under {parent}" + ); + } + let under = |s: u32| -> Vec { + records + .iter() + .filter(|(labels, _)| labels[service] == s) + .map(|(labels, _)| labels[endpoint] % 25) + .collect() + }; + let shares = http.label_shares(endpoint); + for s in [0, 3] { + let children = under(s); + for (i, &expected) in shares.iter().enumerate().take(3) { + let actual = children.iter().filter(|&&c| c == i as u32).count() as f64 + / children.len() as f64; + assert!( + within(actual, expected, children.len()), + "service {s} endpoint {i}" + ); + } + } + } + + #[test] + fn burst_records_hit_the_burst_value_from_their_own_keys() { + let flows = metric("flows", &flows_spec()); + let n = 400_000; + let records = records(&flows, n); + let (keys, burst_keys) = (1_000_000.0, 10_000.0); + let burst: Vec<_> = records.iter().filter(|(_, v)| v[0] >= keys).collect(); + assert!( + within(burst.len() as f64 / n as f64, 0.05, n as usize), + "{}", + burst.len() + ); + assert!(burst.iter().all(|(labels, _)| labels[0] == 999)); + assert!(burst + .iter() + .all(|(_, v)| v[0] < keys + burst_keys && v[0].fract() == 0.0)); + // About 20,000 draws over 10,000 keys see 1 - e^-2 of them. + let distinct: HashSet = burst.iter().map(|(_, v)| v[0] as u64).collect(); + assert!( + (8_400..=8_900).contains(&distinct.len()), + "{}", + distinct.len() + ); + // The background is uniform over its keys and every subnet. + let background: Vec<_> = records.iter().filter(|(_, v)| v[0] < keys).collect(); + let mean = background.iter().map(|(_, v)| v[0]).sum::() / background.len() as f64; + assert!((mean / keys - 0.5).abs() < 0.01, "{mean}"); + let subnet_999 = records.iter().filter(|(l, _)| l[0] == 999).count() as f64 / n as f64; + assert!( + within(subnet_999, flows.label_shares(0)[999], n as usize), + "{subnet_999}" + ); + } + + #[test] + fn latency_is_scaled_by_service_and_the_anomaly_endpoint() { + let http = metric("http", &http_spec()); + let (service, endpoint) = (1, 2); + let factors: Vec = http.label_values[service] + .iter() + .map(|v| scale_factor(v, 42)) + .collect(); + assert!(factors.iter().all(|f| (1.0..=10.0).contains(f))); + assert!(factors.windows(2).any(|w| w[0] != w[1])); + assert_ne!(scale_factor("3", 42), scale_factor("3", 43)); + // Every latency is exactly the Pareto draw of its stream, times its + // service's factor, times 10 on the first endpoint ("0.0"). + let latency_stream = http.labels.len() as u64; + for (index, (labels, values)) in records(&http, 20_000).iter().enumerate() { + let u = unit_uniform(http.seed, latency_stream, index as u64); + let anomaly = if http.label_values[endpoint][labels[endpoint] as usize] == "0.0" { + 10.0 + } else { + 1.0 + }; + let expected = + PARETO_SCALE * u.powf(-0.5) * factors[labels[service] as usize] * anomaly; + assert_eq!(values[0], expected); + } + } + + #[test] + fn keys_follow_zipf_ranks() { + let http = metric("http", &http_spec()); + let n = 200_000; + let user_ids: Vec = records(&http, n).iter().map(|(_, v)| v[1]).collect(); + assert!(user_ids.iter().all(|&k| k < 1e6 && k.fract() == 0.0)); + let shares = zipf_shares(1_000_000, 0.8); + for key in [0.0, 1.0, 9.0] { + let actual = user_ids.iter().filter(|&&k| k == key).count() as f64 / n as f64; + assert!( + within(actual, shares[key as usize], n as usize), + "key {key}: {actual}" + ); + } + } + + #[test] + fn a_tick_spreads_its_records_over_slots_ending_in_the_second() { + // 250 records/s: slots of 2 or 3 records. + let m = RecordMetric::from_table("http", &http_spec(), Some(250), 42).unwrap(); + let tick = m.tick(2, 10_000); + assert_eq!(tick.indices, 500..750); + let mut per_timestamp: BTreeMap = BTreeMap::new(); + for index in tick.indices { + *per_timestamp + .entry(m.timestamp_ms(index, 10_000)) + .or_default() += 1; + } + // Slot s of tick 2 is at 12_000 + 10 · (s + 1): within (12 s, 13 s]. + assert!(per_timestamp + .keys() + .copied() + .eq((1..=100).map(|s| 12_000 + 10 * s))); + assert!(per_timestamp.values().all(|n| (2..=3).contains(n))); + // The first record of the next tick starts the next second's slots. + assert_eq!(m.timestamp_ms(750, 10_000), 13_010); + } + + #[test] + fn records_depend_only_on_seed_name_and_index() { + let a = metric("http", &http_spec()); + assert_eq!( + records(&a, 100), + records(&metric("http", &http_spec()), 100) + ); + // Another metric of the same shape draws other records. + assert_ne!( + records(&a, 100), + records(&metric("data_1_http", &http_spec()), 100) + ); + let reseeded = RecordMetric::from_table("http", &http_spec(), None, 43).unwrap(); + assert_ne!(records(&a, 100), records(&reseeded, 100)); + } + + #[test] + fn share_checks_pass_and_catch_a_wrong_skew() { + for spec in [http_spec(), flows_spec()] { + let report = metric("m", &spec).share_checks(200_000); + assert!(report.checks.iter().all(ShareCheck::holds), "{report:?}"); + } + let report = metric("flows", &flows_spec()).share_checks(200_000); + let burst = report.checks.iter().find(|c| c.label == "burst").unwrap(); + assert_eq!(burst.expected, 0.05); + assert!(report + .checks + .iter() + .any(|c| c.label == "dst_subnet" && c.value == "999")); + // About 10,000 burst draws over 10,000 keys see 1 - 1/e of them. + let (seen, keys) = report.burst_keys.unwrap(); + assert!((6_000..=6_600).contains(&seen) && keys == 10_000, "{seen}"); + + // Records drawn with a flatter service skew than described. + let mut wrong = metric("http", &http_spec()); + wrong.label_zipfs[1] = Zipf::new(25, 0.5); + let report = wrong.share_checks(200_000); + let service_0 = report + .checks + .iter() + .find(|c| c.label == "service" && c.value == "0") + .unwrap(); + assert!(!service_0.holds(), "{service_0:?}"); + } + + #[test] + fn rejects_malformed_records_specs() { + let err = |spec: Value| -> String { + format!( + "{:#}", + RecordMetric::from_table("m", &spec, None, 1).unwrap_err() + ) + }; + let mut spec = http_spec(); + spec["labels"][0].as_object_mut().unwrap().remove("skew"); + assert!(err(spec).contains("needs a `skew`")); + + let mut spec = http_spec(); + spec["values"]["latency"]["scaled_by"] = json!("nope"); + assert!(err(spec).contains("no label `nope`")); + + let mut spec = http_spec(); + spec["values"]["latency"]["zipf"] = json!(1.0); + assert!(err(spec).contains("must be either")); + + let mut spec = flows_spec(); + spec["values"]["src_ip"]["burst"]["value"] = json!(1000); + assert!(err(spec).contains("not a value of `dst_subnet`")); + + let mut spec = flows_spec(); + spec["values"]["dst_port"] = json!({"zipf": 0.0, "keys": 5}); + assert!(err(spec).contains("not be `ts` or a label")); + + let mut spec = flows_spec(); + spec["values"]["src_ip"]["keys"] = json!(1u64 << 53); + assert!(err(spec).contains("all keys ≤ 2^53")); + + assert!(RecordMetric::from_table("m", &http_spec(), Some(0), 1).is_err()); + } +} diff --git a/asap-tools/data-sources/paced-feeder/src/table.rs b/asap-tools/data-sources/paced-feeder/src/table.rs index e6f73329..8454f8b5 100644 --- a/asap-tools/data-sources/paced-feeder/src/table.rs +++ b/asap-tools/data-sources/paced-feeder/src/table.rs @@ -6,13 +6,16 @@ //! its parent's value, a `.`, and the child index (`job="3"`, //! `label_0="3.17"`), so a child value names its parent and is unique across //! parents. Labels without `child_of` form a cross product, and the metric -//! has the product of all cardinalities as series. +//! has the product of all cardinalities as series. `records` metrics are +//! generated by [`crate::records`]. +use crate::records::RecordMetric; use crate::workload::{splitmix64, Kind, LabelSpec, Metric, Workload}; use anyhow::{bail, ensure, Context, Result}; use serde::Deserialize; use serde_json::Value; use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; #[derive(Deserialize)] struct Table { @@ -67,11 +70,13 @@ struct CounterValues { } /// The metrics of `workloads[0].e2e.metrics` in the table at `path`, in name -/// order, or only `only`. `samples_per_sec` overrides every metric's rate. +/// order, or only `only`. `samples_per_sec` overrides every gauge and +/// counter's rate, `records_per_sec` every records metric's. pub fn load( path: &str, only: Option<&str>, samples_per_sec: Option, + records_per_sec: Option, seed: u64, ) -> Result { let text = std::fs::read_to_string(path).with_context(|| format!("reading {path}"))?; @@ -81,23 +86,52 @@ pub fn load( bail!("{path}: `workloads` is empty"); }; let mut metrics = Vec::new(); + let mut records = Vec::new(); let mut rates = Vec::new(); for (name, spec) in &first.e2e.metrics { if only.is_some_and(|only| only != name) { continue; } - let (metric, rate) = - metric(name, spec, seed).with_context(|| format!("metric `{name}`"))?; + let context = || format!("metric `{name}`"); + ensure!( + is_identifier(name), + "metric `{name}`: the name must match [A-Za-z_][A-Za-z0-9_]* (it is also the ClickHouse table)" + ); + if spec.get("type").and_then(Value::as_str) == Some("records") { + let metric = RecordMetric::from_table(name, spec, records_per_sec, seed) + .with_context(context)?; + records.push(Arc::new(metric)); + continue; + } + let (metric, rate) = metric(name, spec, seed).with_context(context)?; metrics.push(metric); rates.push(rate); } if let Some(only) = only { - ensure!(!metrics.is_empty(), "{path}: no metric `{only}`"); + ensure!( + !metrics.is_empty() || !records.is_empty(), + "{path}: no metric `{only}`" + ); } ensure!( - !metrics.is_empty(), + !metrics.is_empty() || !records.is_empty(), "{path}: workloads[0].e2e.metrics is empty" ); + ensure!( + !metrics.is_empty() || samples_per_sec.is_none(), + "--samples-per-sec applies to gauges and counters; use --records-per-sec for records" + ); + ensure!( + !records.is_empty() || records_per_sec.is_none(), + "--records-per-sec applies to records metrics only" + ); + if metrics.is_empty() { + return Ok(Workload { + samples_per_sec: 0, + metrics, + records, + }); + } let samples_per_sec = match samples_per_sec { Some(rate) => rate, // One feeder sends every metric at one rate. @@ -110,19 +144,22 @@ pub fn load( Ok(Workload { samples_per_sec, metrics, + records, }) } fn metric(name: &str, spec: &Value, seed: u64) -> Result<(Metric, u32)> { - ensure!( - is_identifier(name), - "the name must match [A-Za-z_][A-Za-z0-9_]* (it is also the ClickHouse table)" - ); let kind = spec.get("type").and_then(Value::as_str).unwrap_or_default(); if kind != "gauge" && kind != "counter" { - bail!("type `{kind}` is not supported; the feeder generates `gauge` and `counter` metrics"); + bail!("type `{kind}` is not supported; the feeder generates `gauge`, `counter` and `records` metrics"); } let spec: MetricSpec = serde_json::from_value(spec.clone())?; + if let Some(label) = spec.labels.iter().find(|l| l.skew.is_some()) { + bail!( + "label `{}`: `skew` applies to records metrics only; series skew is the value's `zipf`", + label.name + ); + } let series = series_labels(&spec.labels)?; let (skew, kind) = if spec.kind == "gauge" { let values: GaugeValues = serde_json::from_value(spec.values).context("gauge `values`")?; @@ -164,8 +201,41 @@ fn metric(name: &str, spec: &Value, seed: u64) -> Result<(Metric, u32)> { /// Each series' label values, in `labels` order. Series are numbered in mixed /// radix with the last label varying fastest. pub fn series_labels(labels: &[LabelSpec]) -> Result>> { + let parents = label_parents(labels)?; + let num_series = labels + .iter() + .try_fold(1usize, |n, l| { + n.checked_mul(usize::try_from(l.cardinality).ok()?) + }) + .context("too many series")?; + + let mut series = Vec::with_capacity(num_series); + let mut index = vec![0u64; labels.len()]; + for _ in 0..num_series { + let mut values: Vec = Vec::with_capacity(labels.len()); + for (i, parent) in parents.iter().enumerate() { + values.push(match parent { + None => index[i].to_string(), + Some(p) => format!("{}.{}", values[*p], index[i]), + }); + } + series.push(values); + // Advance the mixed-radix index, last label fastest. + for i in (0..labels.len()).rev() { + index[i] += 1; + if index[i] < labels[i].cardinality { + break; + } + index[i] = 0; + } + } + Ok(series) +} + +/// Validates `labels` and returns the index of each label's parent, which +/// must come earlier. +pub fn label_parents(labels: &[LabelSpec]) -> Result>> { ensure!(!labels.is_empty(), "a metric needs at least one label"); - // Index of each label's parent, which must come earlier. let mut parents = Vec::with_capacity(labels.len()); for (i, label) in labels.iter().enumerate() { let name = &label.name; @@ -194,34 +264,7 @@ pub fn series_labels(labels: &[LabelSpec]) -> Result>> { ), }); } - let num_series = labels - .iter() - .try_fold(1usize, |n, l| { - n.checked_mul(usize::try_from(l.cardinality).ok()?) - }) - .context("too many series")?; - - let mut series = Vec::with_capacity(num_series); - let mut index = vec![0u64; labels.len()]; - for _ in 0..num_series { - let mut values: Vec = Vec::with_capacity(labels.len()); - for (i, parent) in parents.iter().enumerate() { - values.push(match parent { - None => index[i].to_string(), - Some(p) => format!("{}.{}", values[*p], index[i]), - }); - } - series.push(values); - // Advance the mixed-radix index, last label fastest. - for i in (0..labels.len()).rev() { - index[i] += 1; - if index[i] < labels[i].cardinality { - break; - } - index[i] = 0; - } - } - Ok(series) + Ok(parents) } /// Each series' value scale for Zipf(`theta`) skew over the series: the @@ -306,7 +349,7 @@ fn closed_cardinality(labels: &[LabelSpec], subset: &[usize]) -> u64 { .fold(1u64, u64::saturating_mul) } -fn is_identifier(name: &str) -> bool { +pub fn is_identifier(name: &str) -> bool { let mut chars = name.chars(); chars .next() @@ -315,7 +358,7 @@ fn is_identifier(name: &str) -> bool { } /// 64-bit FNV-1a, to derive a per-name seed. -fn fnv1a(s: &str) -> u64 { +pub fn fnv1a(s: &str) -> u64 { s.bytes().fold(0xcbf2_9ce4_8422_2325, |h, b| { (h ^ u64::from(b)).wrapping_mul(0x0000_0100_0000_01b3) }) @@ -331,6 +374,7 @@ mod tests { name: name.to_string(), cardinality, child_of: child_of.map(str::to_string), + skew: None, } } @@ -458,7 +502,7 @@ mod tests { #[test] fn loads_gauge_and_counter_from_a_table() { let path = write_table(classic_metrics()); - let w = load(&path, None, None, 1).unwrap(); + let w = load(&path, None, None, None, 1).unwrap(); assert_eq!(w.samples_per_sec, 4); assert_eq!(w.num_series(), 12); let (data, total) = (&w.metrics[0], &w.metrics[1]); @@ -478,22 +522,92 @@ mod tests { assert_eq!(data.scales, total.scales); assert_ne!(data.seed, total.seed); - let only = load(&path, Some("data_total"), Some(2), 1).unwrap(); + let only = load(&path, Some("data_total"), Some(2), None, 1).unwrap(); assert_eq!(only.metrics.len(), 1); assert_eq!(only.samples_per_sec, 2); - assert!(load(&path, Some("nope"), None, 1).is_err()); + assert!(load(&path, Some("nope"), None, None, 1).is_err()); } #[test] - fn rejects_records_and_mismatched_keys() { - let path = write_table(json!({"http": {"type": "records", "labels": [], "rate": 1}})); - let err = format!("{:#}", load(&path, None, None, 1).unwrap_err()); - assert!(err.contains("type `records` is not supported"), "{err}"); - + fn rejects_mismatched_keys() { let mut metrics = classic_metrics(); metrics["data"]["values"]["value"]["keys"] = json!(7); let path = write_table(metrics); - let err = format!("{:#}", load(&path, Some("data"), None, 1).unwrap_err()); + let err = format!( + "{:#}", + load(&path, Some("data"), None, None, 1).unwrap_err() + ); assert!(err.contains("must equal the number of series"), "{err}"); } + + fn all_metrics() -> Value { + let mut metrics = classic_metrics(); + metrics["http"] = crate::records::tests::http_spec(); + metrics["flows"] = crate::records::tests::flows_spec(); + metrics + } + + #[test] + fn loads_records_metrics() { + let mut metrics = all_metrics(); + metrics + .as_object_mut() + .unwrap() + .retain(|_, m| m["type"] == "records"); + let path = write_table(metrics); + let w = load(&path, None, None, None, 1).unwrap(); + assert!(w.metrics.is_empty()); + assert_eq!(w.names(), ["flows", "http"]); + assert_eq!(w.rows_per_tick(), 6_000_000); + let scaled = load(&path, Some("http"), None, Some(500), 1).unwrap(); + assert_eq!(scaled.names(), ["http"]); + assert_eq!(scaled.records[0].records_per_sec, 500); + assert_eq!(scaled.records[0].fields, ["latency", "user_id"]); + } + + #[test] + fn one_feeder_sends_both_kinds_of_metric() { + let path = write_table(all_metrics()); + let both = load(&path, None, Some(2), Some(500), 1).unwrap(); + assert_eq!((both.metrics.len(), both.records.len()), (2, 2)); + assert_eq!(both.samples_per_sec, 2); + assert!(both.records.iter().all(|r| r.records_per_sec == 500)); + // `--metric` picks one of either kind. + assert_eq!( + load(&path, Some("http"), None, None, 1) + .unwrap() + .records + .len(), + 1 + ); + assert_eq!( + load(&path, Some("data"), None, None, 1) + .unwrap() + .metrics + .len(), + 1 + ); + // Each rate override applies to its own kind only. + assert!(load(&path, Some("http"), Some(10), None, 1).is_err()); + assert!(load(&path, Some("data"), None, Some(10), 1).is_err()); + } + + #[test] + fn rejects_unknown_types_and_skew_on_series_labels() { + let path = write_table(json!({"x": {"type": "histogram", "labels": []}})); + let err = format!("{:#}", load(&path, None, None, None, 1).unwrap_err()); + assert!(err.contains("type `histogram` is not supported"), "{err}"); + + let mut metrics = classic_metrics(); + metrics["data"]["labels"][0]["skew"] = json!(1.0); + let path = write_table(metrics); + let err = format!( + "{:#}", + load(&path, Some("data"), None, None, 1).unwrap_err() + ); + assert!( + err.contains("`skew` applies to records metrics only"), + "{err}" + ); + } } diff --git a/asap-tools/data-sources/paced-feeder/src/workload.rs b/asap-tools/data-sources/paced-feeder/src/workload.rs index de8f5d31..20567c85 100644 --- a/asap-tools/data-sources/paced-feeder/src/workload.rs +++ b/asap-tools/data-sources/paced-feeder/src/workload.rs @@ -1,11 +1,14 @@ -//! Seeded synthetic workload: one or more metrics, each a set of labelled -//! series emitting `samples_per_sec` evenly spaced samples per second. +//! Seeded synthetic workload: either one or more metrics, each a set of +//! labelled series emitting `samples_per_sec` evenly spaced samples per +//! second, or one or more `records` metrics (see [`crate::records`]). //! //! A sample's value depends only on `(metric seed, series, sample index)`, //! never on wall-clock time or on which sink sends it, so two feeders started //! with the same parameters produce identical data. +use crate::records::{RecordMetric, Records}; use serde::Deserialize; +use std::sync::Arc; /// One generated sample of one metric. `series` indexes into [`Metric::series`]. #[derive(Debug, Clone, Copy, PartialEq)] @@ -15,8 +18,25 @@ pub struct Row { pub value: f64, } +/// One tick's rows of one metric: samples of series, or records. +#[derive(Debug, Clone, PartialEq)] +pub enum Rows { + Samples(Vec), + Records(Records), +} + +impl Rows { + pub fn len(&self) -> usize { + match self { + Rows::Samples(rows) => rows.len(), + Rows::Records(records) => records.len(), + } + } +} + /// A label with `cardinality` values, or, with `child_of`, that many values -/// under each value of the parent label. +/// under each value of the parent label. `skew` (records metrics only) is the +/// Zipf exponent its values are drawn with. #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(deny_unknown_fields)] pub struct LabelSpec { @@ -24,6 +44,8 @@ pub struct LabelSpec { pub cardinality: u64, #[serde(default)] pub child_of: Option, + #[serde(default)] + pub skew: Option, } #[derive(Debug, Clone, PartialEq)] @@ -51,10 +73,14 @@ pub struct Metric { pub seed: u64, } +/// Gauge and counter `metrics`, or `records` metrics, never both: they have +/// different sub-second slots. #[derive(Debug, Clone)] pub struct Workload { + /// Samples per second per series of `metrics`; unused for `records`. pub samples_per_sec: u32, pub metrics: Vec, + pub records: Vec>, } impl Workload { @@ -82,6 +108,7 @@ impl Workload { name: name.to_string(), cardinality: cardinality as u64, child_of: None, + skew: None, }; let metric = Metric { name: name.to_string(), @@ -100,6 +127,7 @@ impl Workload { Self { samples_per_sec, metrics: vec![metric], + records: Vec::new(), } } @@ -107,15 +135,42 @@ impl Workload { self.metrics.iter().map(|m| m.series.len()).sum() } + /// Series samples or records per second. pub fn rows_per_tick(&self) -> usize { self.num_series() * self.samples_per_sec as usize + + self + .records + .iter() + .map(|r| r.records_per_sec as usize) + .sum::() + } + + /// The metric names, gauges and counters first, then records. + pub fn names(&self) -> Vec { + let series = self.metrics.iter().map(|m| m.name.clone()); + series + .chain(self.records.iter().map(|r| r.name.clone())) + .collect() + } + + /// Rows of the `target`-th metric of [`Self::names`] for one-second tick + /// `tick`, with event time in `(t, t + 1 s]`, where `t` is + /// `start_ms + tick · 1 s`: the same left-open interval as ASAPQuery's + /// one-second windows, so a tick never spans two of them. + pub fn tick_rows(&self, target: usize, tick: u64, start_ms: i64) -> Rows { + match target.checked_sub(self.metrics.len()) { + None => Rows::Samples( + (0..self.samples_per_sec) + .flat_map(|slot| self.series_rows(target, tick, slot, start_ms)) + .collect(), + ), + Some(r) => Rows::Records(self.records[r].tick(tick, start_ms)), + } } - /// Samples of `metric` for sub-second slot `slot` of one-second tick - /// `tick`, one per series. Slots are spaced evenly over `(t, t + 1 s]`, - /// where `t` is the tick's start: the same left-open interval as - /// ASAPQuery's one-second windows, so a tick never spans two of them. - pub fn slot_rows(&self, metric: usize, tick: u64, slot: u32, start_ms: i64) -> Vec { + /// Samples of `metric` for sub-second slot `slot` of tick `tick`, one per + /// series. Slot `j` of `n` is at `t + (j + 1) · 1 s / n`. + pub fn series_rows(&self, metric: usize, tick: u64, slot: u32, start_ms: i64) -> Vec { let per_sec = u64::from(self.samples_per_sec); let slot = u64::from(slot); let timestamp_ms = start_ms + tick as i64 * 1_000 + ((slot + 1) * 1_000 / per_sec) as i64; @@ -151,7 +206,7 @@ impl Metric { /// Counter-based uniform draw in (0, 1]: a stateless hash of the inputs, so /// any sample can be regenerated independently of the ones before it. -fn unit_uniform(seed: u64, series: u64, sample_index: u64) -> f64 { +pub fn unit_uniform(seed: u64, series: u64, sample_index: u64) -> f64 { let x = splitmix64(seed ^ splitmix64(series ^ splitmix64(sample_index))); // Top 53 bits, shifted into (0, 1] so the Pareto transform never divides by zero. ((x >> 11) + 1) as f64 / (1u64 << 53) as f64 @@ -176,7 +231,7 @@ mod tests { fn each_slot_has_every_series_at_its_timestamp() { let w = workload(); for (slot, timestamp_ms) in [(0, 15_500), (1, 16_000)] { - let rows = w.slot_rows(0, 5, slot, 10_000); + let rows = w.series_rows(0, 5, slot, 10_000); assert_eq!(rows.len(), 6); assert!(rows.iter().all(|r| r.timestamp_ms == timestamp_ms)); assert!(rows.iter().map(|r| r.series).eq(0..6)); @@ -187,7 +242,7 @@ mod tests { fn values_depend_only_on_seed_series_and_sample_index() { let w = workload(); let values = |w: &Workload, start_ms| -> Vec { - w.slot_rows(0, 4, 1, start_ms) + w.series_rows(0, 4, 1, start_ms) .iter() .map(|r| r.value) .collect() From 581e5d9cca47cb3fd9eb4a5a65b88fe2914d5adf Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 13 Oct 2026 02:02:57 +0000 Subject: [PATCH 4/8] test(asap-tools): describe the grouped workload's values without history Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AnJ7LocWxguh3q4QWkCYae --- asap-tools/data-sources/paced-feeder/src/workload.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/asap-tools/data-sources/paced-feeder/src/workload.rs b/asap-tools/data-sources/paced-feeder/src/workload.rs index de8f5d31..d82d02f4 100644 --- a/asap-tools/data-sources/paced-feeder/src/workload.rs +++ b/asap-tools/data-sources/paced-feeder/src/workload.rs @@ -200,9 +200,9 @@ mod tests { } #[test] - fn grouped_values_keep_the_seeded_pareto_draw() { - // The grouped workload's values are the plain Pareto draw of - // `(seed, series, sample index)`, as before metrics had scales. + fn grouped_values_are_the_seeded_pareto_draw() { + // Every series of the grouped workload has scale 1, so its values are + // the plain Pareto draw of `(seed, series, sample index)`. let w = workload(); let u = unit_uniform(7, 3, 9); assert_eq!(w.metrics[0].value(3, 9), u.powf(-1.0 / 1.5)); From 0222064e313e20e0c6def734a3812a9dddfc774d Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 13 Oct 2026 02:04:11 +0000 Subject: [PATCH 5/8] fix(asap-tools): cap Zipf sizes and check records' label coverage cheaply A Zipf keeps a dense CDF and a u32 guide entry per value, so a records label's cardinality and a field's keys are capped at 1e8 (1.2 GB), with an error naming the cap; burst keys are drawn uniformly and keep only the 2^53 bound. `--verify-facts` no longer enumerates every label combination of a records metric: per label subset it prints the possible tuples and how many the sampled 1e6 records hold, failing only if they hold more. The share checks are unchanged. The `Workload` doc now says one feeder sends both kinds of metric. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AnJ7LocWxguh3q4QWkCYae --- .../data-sources/paced-feeder/README.md | 14 ++-- .../data-sources/paced-feeder/src/main.rs | 27 +++--- .../data-sources/paced-feeder/src/records.rs | 84 +++++++++++++++++-- .../data-sources/paced-feeder/src/table.rs | 2 +- .../data-sources/paced-feeder/src/workload.rs | 3 +- 5 files changed, 104 insertions(+), 26 deletions(-) diff --git a/asap-tools/data-sources/paced-feeder/README.md b/asap-tools/data-sources/paced-feeder/README.md index fea7a119..3340763f 100644 --- a/asap-tools/data-sources/paced-feeder/README.md +++ b/asap-tools/data-sources/paced-feeder/README.md @@ -120,9 +120,10 @@ name), `n` and the draw, so records match sketch-bench's `label_shares()` sending anything: for every non-empty subset of a metric's labels it prints the expected number of distinct label-value tuples (the product of the cardinalities of the subset plus every label's `child_of` ancestors) and the -number among the generated series (for records metrics, among all label-value -combinations a record can have), and exits non-zero if any differ. For records -metrics it also draws records `0..1e6` and compares each label's three +number among the generated series, and exits non-zero if any differ. For +records metrics it draws records `0..1e6`, prints how many of the possible +tuples of each subset they hold (skew leaves rare ones unseen; more than +possible fails), and compares each label's three heaviest values' shares (a child's index pooled over parents, i.e. its share given the parent), the burst value's share and the share of burst records with `label_shares()`; a share fails if it is off by more than 5 standard errors, @@ -139,8 +140,9 @@ data_total: 10000 series … $ paced_feeder --workload-table synthetic-templatesmultigroup-shared1.json --verify-facts -flows: 4000000 records/s, 3000000 label combinations - labels expected generated +flows: 4000000 records/s + label-value tuples among 1000000 records + labels possible seen dst_subnet 1000 1000 … shares of 1000000 records (tolerance 5 standard errors) @@ -151,7 +153,7 @@ flows: 4000000 records/s, 3000000 label combinations … burst records 0.05000 0.05029 0.00109 burst keys: 9938 distinct of 10000 -http: 2000000 records/s, 10000 label combinations +http: 2000000 records/s … endpoint *.0 (given service) 0.29841 0.29855 0.00229 … diff --git a/asap-tools/data-sources/paced-feeder/src/main.rs b/asap-tools/data-sources/paced-feeder/src/main.rs index c466cf15..32413d05 100644 --- a/asap-tools/data-sources/paced-feeder/src/main.rs +++ b/asap-tools/data-sources/paced-feeder/src/main.rs @@ -314,16 +314,23 @@ fn verify_facts(workload: &Workload) -> Result { ok &= print_label_facts(&metric.labels, &metric.series); } for metric in &workload.records { - // Every combination of label values a record can have. - let domain = table::series_labels(&metric.labels)?; - println!( - "{}: {} records/s, {} label combinations", - metric.name, - metric.records_per_sec, - domain.len() - ); - ok &= print_label_facts(&metric.labels, &domain); - let report = metric.share_checks(records::SHARE_SAMPLE_RECORDS); + let n = records::SHARE_SAMPLE_RECORDS; + println!("{}: {} records/s", metric.name, metric.records_per_sec); + println!(" label-value tuples among {n} records"); + println!(" {:<40} {:>12} {:>12}", "labels", "possible", "seen"); + for fact in metric.label_coverage(n) { + // Every tuple a record has is possible; skew leaves some unseen. + let holds = fact.actual <= fact.expected; + ok &= holds; + println!( + " {:<40} {:>12} {:>12}{}", + fact.labels.join(","), + fact.expected, + fact.actual, + if holds { "" } else { " MISMATCH" } + ); + } + let report = metric.share_checks(n); println!( " shares of {} records (tolerance {} standard errors)", report.records, diff --git a/asap-tools/data-sources/paced-feeder/src/records.rs b/asap-tools/data-sources/paced-feeder/src/records.rs index 86148c10..7075d17b 100644 --- a/asap-tools/data-sources/paced-feeder/src/records.rs +++ b/asap-tools/data-sources/paced-feeder/src/records.rs @@ -27,7 +27,7 @@ //! record's value of `L` is its first one (`0`, or `0.0` for a child: the //! heaviest). Scaling keeps the tail index `a`. -use crate::table::{fnv1a, is_identifier, label_parents}; +use crate::table::{closed_cardinality, fnv1a, is_identifier, label_parents, Fact}; use crate::workload::{splitmix64, unit_uniform, LabelSpec}; use anyhow::{bail, ensure, Context, Result}; use serde::Deserialize; @@ -45,6 +45,9 @@ pub const PARETO_SCALE: f64 = 1_000.0; pub const SCALE_RANGE: [f64; 2] = [1.0, 10.0]; /// Keys are sent as `Float64`, which holds integers exactly up to 2^53. const MAX_KEY: u64 = 1 << 53; +/// The most values a Zipf draws from (a label's values or a field's keys): it +/// keeps a CDF entry and a `u32` guide entry per value, 1.2 GB at this cap. +pub const MAX_ZIPF_VALUES: u64 = 100_000_000; /// The facts check counts shares over this many records. pub const SHARE_SAMPLE_RECORDS: u64 = 1_000_000; /// A share's check fails if it is off by more than this many standard errors @@ -113,6 +116,7 @@ pub fn zipf_shares(n: u64, theta: f64) -> Vec { impl Zipf { pub fn new(n: u64, theta: f64) -> Self { + assert!(n <= MAX_ZIPF_VALUES, "a Zipf over {n} values"); let mut cdf: Vec = zipf_shares(n, theta) .into_iter() .scan(0.0, |sum, p| { @@ -231,9 +235,10 @@ impl RecordMetric { .with_context(|| format!("label `{}` needs a `skew`", label.name))?; ensure!(skew >= 0.0, "label `{}`: skew must be ≥ 0", label.name); ensure!( - label.cardinality <= u64::from(u32::MAX), - "label `{}`: too many values", - label.name + label.cardinality <= MAX_ZIPF_VALUES, + "label `{}`: cardinality {} is over the {MAX_ZIPF_VALUES} values a Zipf draws from", + label.name, + label.cardinality ); label_zipfs.push(Zipf::new(label.cardinality, skew)); } @@ -259,8 +264,8 @@ impl RecordMetric { } => { ensure!(theta >= 0.0, "value `{field}`: zipf must be ≥ 0"); ensure!( - keys > 0 && keys <= MAX_KEY, - "value `{field}`: keys must be in [1, 2^53]" + keys > 0 && keys <= MAX_ZIPF_VALUES, + "value `{field}`: keys must be in [1, {MAX_ZIPF_VALUES}], the values a Zipf draws from" ); if let Some(b) = field_burst { ensure!(burst.is_none(), "at most one value may have a burst"); @@ -438,6 +443,42 @@ impl RecordMetric { shares } + /// For every non-empty subset of the labels, the number of label-value + /// tuples a record can have (as for series: closed under `child_of`) and + /// the distinct tuples among records `0..n`. A child's value index names + /// its parent, so the tuples need not hold the ancestors. + pub fn label_coverage(&self, n: u64) -> Vec { + let num_labels = self.labels.len(); + assert!( + num_labels < 32, + "too many labels to enumerate their subsets" + ); + let (mut labels, mut values) = (vec![0u32; num_labels], vec![0.0; self.fields.len()]); + let mut drawn = Vec::with_capacity(n as usize * num_labels); + for index in 0..n { + self.record(index, &mut labels, &mut values); + drawn.extend_from_slice(&labels); + } + (1u32..1 << num_labels) + .map(|mask| { + let subset: Vec = (0..num_labels).filter(|i| mask & (1 << i) != 0).collect(); + let actual = drawn + .chunks(num_labels) + .map(|record| subset.iter().map(|&i| record[i]).collect::>()) + .collect::>() + .len() as u64; + Fact { + labels: subset + .iter() + .map(|&i| self.labels[i].name.clone()) + .collect(), + expected: closed_cardinality(&self.labels, &subset), + actual, + } + }) + .collect() + } + /// The share checks of the facts check over records `0..n`: each label's /// heaviest values (a child's index pooled over parents, so its share given /// the parent), the burst value and the share of burst records. @@ -814,6 +855,27 @@ pub mod tests { assert_ne!(records(&a, 100), records(&reseeded, 100)); } + #[test] + fn label_coverage_counts_distinct_tuples_within_the_domain() { + let coverage = metric("http", &http_spec()).label_coverage(200_000); + assert_eq!(coverage.len(), 15); + let by_labels = |names: &str| { + let fact = coverage + .iter() + .find(|f| f.labels.join(",") == names) + .unwrap(); + (fact.expected, fact.actual) + }; + // Small domains are covered; an endpoint names its service. + assert_eq!(by_labels("region"), (4, 4)); + assert_eq!(by_labels("endpoint"), (625, 625)); + assert_eq!(by_labels("service,endpoint"), (625, 625)); + // Heavy skew leaves some of the 10,000 full tuples unseen. + let (expected, actual) = by_labels("region,service,endpoint,status"); + assert!(expected == 10_000 && actual < expected, "{actual}"); + assert!(coverage.iter().all(|f| f.actual <= f.expected)); + } + #[test] fn share_checks_pass_and_catch_a_wrong_skew() { for spec in [http_spec(), flows_spec()] { @@ -871,8 +933,16 @@ pub mod tests { spec["values"]["dst_port"] = json!({"zipf": 0.0, "keys": 5}); assert!(err(spec).contains("not be `ts` or a label")); + // A Zipf's tables are capped; burst keys are drawn uniformly, so only + // their numbering is. + let mut spec = flows_spec(); + spec["values"]["src_ip"]["keys"] = json!(MAX_ZIPF_VALUES + 1); + assert!(err(spec).contains("keys must be in [1, 100000000]")); + let mut spec = http_spec(); + spec["labels"][0]["cardinality"] = json!(MAX_ZIPF_VALUES + 1); + assert!(err(spec).contains("cardinality 100000001 is over")); let mut spec = flows_spec(); - spec["values"]["src_ip"]["keys"] = json!(1u64 << 53); + spec["values"]["src_ip"]["burst"]["keys"] = json!(1u64 << 53); assert!(err(spec).contains("all keys ≤ 2^53")); assert!(RecordMetric::from_table("m", &http_spec(), Some(0), 1).is_err()); diff --git a/asap-tools/data-sources/paced-feeder/src/table.rs b/asap-tools/data-sources/paced-feeder/src/table.rs index 8454f8b5..97cd6546 100644 --- a/asap-tools/data-sources/paced-feeder/src/table.rs +++ b/asap-tools/data-sources/paced-feeder/src/table.rs @@ -333,7 +333,7 @@ pub fn facts(labels: &[LabelSpec], series: &[Vec]) -> Vec { /// The product of the cardinalities of `subset` and every ancestor of its /// labels: a child value names its parent, so grouping by a child groups by /// its ancestors too. -fn closed_cardinality(labels: &[LabelSpec], subset: &[usize]) -> u64 { +pub fn closed_cardinality(labels: &[LabelSpec], subset: &[usize]) -> u64 { let mut closed = HashSet::new(); let mut todo: Vec = subset.to_vec(); while let Some(i) = todo.pop() { diff --git a/asap-tools/data-sources/paced-feeder/src/workload.rs b/asap-tools/data-sources/paced-feeder/src/workload.rs index 20567c85..4649d86e 100644 --- a/asap-tools/data-sources/paced-feeder/src/workload.rs +++ b/asap-tools/data-sources/paced-feeder/src/workload.rs @@ -73,8 +73,7 @@ pub struct Metric { pub seed: u64, } -/// Gauge and counter `metrics`, or `records` metrics, never both: they have -/// different sub-second slots. +/// Gauge and counter `metrics` and `records` metrics, sent by one feeder. #[derive(Debug, Clone)] pub struct Workload { /// Samples per second per series of `metrics`; unused for `records`. From 660a6db21b22fc3a15f2c17e7a854514c684a04a Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 13 Oct 2026 03:09:47 +0000 Subject: [PATCH 6/8] fix(asap-tools): keep each series' samples of a tick in one ordered request Concurrent requests split a tick by time slot, so a series' later sample could arrive first and ASAPQuery's increase accumulator, which reads samples in arrival order, took the lower counter value for a reset. A tick's rows are now series-major and requests are cut only between series (the batch size rounded down to whole series, at least one), so each series' samples of a tick arrive in one request in timestamp order; requests stay concurrent. Prometheus therefore needs no out-of-order window. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AnJ7LocWxguh3q4QWkCYae --- .../data-sources/paced-feeder/README.md | 19 +-- .../data-sources/paced-feeder/src/encode.rs | 4 +- .../data-sources/paced-feeder/src/main.rs | 33 +++--- .../data-sources/paced-feeder/src/workload.rs | 110 ++++++++++++------ 4 files changed, 104 insertions(+), 62 deletions(-) diff --git a/asap-tools/data-sources/paced-feeder/README.md b/asap-tools/data-sources/paced-feeder/README.md index 0ff1d954..73238787 100644 --- a/asap-tools/data-sources/paced-feeder/README.md +++ b/asap-tools/data-sources/paced-feeder/README.md @@ -53,15 +53,18 @@ credentials. ## Pacing and stats -Each tick's rows are split into `--batch-rows` requests and sent with up to -`--concurrency` in flight. The next tick starts only after every request of -the current one returns, so samples arrive out of order only within a tick. -A tick starting at `t` holds samples spaced evenly over `(t, t + 1 s]`, the -same left-open interval as ASAPQuery's one-second windows, so a tick never -spans two windows. Out-of-order samples within it are then safe for +Each tick's rows are split into requests of about `--batch-rows` rows and +sent with up to `--concurrency` in flight. A tick's rows are series-major and +requests are cut only between series, so each series' samples of a tick go +in one request, in timestamp order; the next tick starts only after every +request of the current one returns. Each series' samples therefore arrive in +order (counters never seem to reset, and Prometheus needs no out-of-order +window), while samples of different series of a group may arrive out of +order within a tick. A tick starting at `t` holds samples spaced evenly over +`(t, t + 1 s]`, the same left-open interval as ASAPQuery's one-second +windows, so a tick never spans two windows, and that reordering is safe for ASAPQuery as long as its allowed lateness is at least 1 s (the default is -5 s). Prometheus' remote-write receiver needs an out-of-order window of at -least 1 s (`storage.tsdb.out_of_order_time_window`). +5 s). `--samples-per-sec` is at most 1000, so timestamps (in ms) never repeat. The default batch size is 50,000 rows for remote write, which keeps a compressed request under axum's 2 MB body limit in ASAPQuery's ingest, and 100,000 for diff --git a/asap-tools/data-sources/paced-feeder/src/encode.rs b/asap-tools/data-sources/paced-feeder/src/encode.rs index 32a4caba..880c2202 100644 --- a/asap-tools/data-sources/paced-feeder/src/encode.rs +++ b/asap-tools/data-sources/paced-feeder/src/encode.rs @@ -157,7 +157,7 @@ mod tests { #[test] fn remote_write_decodes_to_one_series_per_row() { let w = workload(); - let rows = w.slot_rows(2, 0, 1_700_000_000_000); + let rows = w.tick_rows(2, 1_700_000_000_000); let body = RemoteWriteEncoder::new(&w, "data").encode(&rows); let raw = snap::raw::Decoder::new().decompress_vec(&body).unwrap(); @@ -191,7 +191,7 @@ mod tests { #[test] fn row_binary_round_trips_every_column() { let w = workload(); - let rows = w.slot_rows(0, 0, 42_000); + let rows = w.tick_rows(0, 42_000); let buf = RowBinaryEncoder::new(&w).encode(&rows); let mut cursor = buf.as_slice(); diff --git a/asap-tools/data-sources/paced-feeder/src/main.rs b/asap-tools/data-sources/paced-feeder/src/main.rs index 8d1ae688..184d85e7 100644 --- a/asap-tools/data-sources/paced-feeder/src/main.rs +++ b/asap-tools/data-sources/paced-feeder/src/main.rs @@ -16,7 +16,7 @@ use serde::Serialize; use std::sync::Arc; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use tokio::sync::Semaphore; -use workload::{Row, Workload}; +use workload::{series_batches, Workload}; #[derive(Debug, Clone, Copy, ValueEnum, Serialize)] #[serde(rename_all = "kebab-case")] @@ -71,9 +71,10 @@ struct Args { #[arg(long)] start_ms: Option, - /// Rows per HTTP request. Defaults to 50,000 for remote write, which keeps - /// a compressed request under axum's 2 MB body limit in ASAPQuery's ingest, - /// and 100,000 for ClickHouse, where each INSERT creates a part. + /// Rows per HTTP request, rounded down to whole series (at least one). + /// Defaults to 50,000 for remote write, which keeps a compressed request + /// under axum's 2 MB body limit in ASAPQuery's ingest, and 100,000 for + /// ClickHouse, where each INSERT creates a part. #[arg(long)] batch_rows: Option, /// Concurrent requests within one tick. @@ -263,25 +264,23 @@ async fn main() -> Result<()> { let started = Instant::now(); let (mut rows, mut bytes) = (0, 0); - // A tick's requests go out concurrently, so a group's samples within - // the tick may arrive out of order. That is safe for ASAPQuery: the - // tick lies in one window (see `slot_rows`) and is under one second - // behind the watermark, within the allowed lateness. - let tick_rows: Arc> = Arc::new( - (0..args.samples_per_sec) - .flat_map(|slot| workload.slot_rows(tick, slot, start_ms)) - .collect(), - ); + // A tick's requests go out concurrently, each holding whole series + // (see `series_batches`): a series' samples arrive in order, but a + // group's samples across series may not. That is safe for ASAPQuery: + // the tick lies in one window (see `tick_rows`) and is under one + // second behind the watermark, within the allowed lateness. + let tick_rows = Arc::new(workload.tick_rows(tick, start_ms)); + let per_series = args.samples_per_sec as usize; let mut tasks = tokio::task::JoinSet::new(); - for begin in (0..tick_rows.len()).step_by(batch_rows) { - let end = (begin + batch_rows).min(tick_rows.len()); + for range in series_batches(tick_rows.len(), per_series, batch_rows) { let (tick_rows, sink) = (Arc::clone(&tick_rows), Arc::clone(&sink)); let permit = Arc::clone(&permits).acquire_owned().await?; tasks.spawn(async move { let _permit = permit; - let body = sink.encoder.encode(&tick_rows[begin..end]); + let rows = range.len(); + let body = sink.encoder.encode(&tick_rows[range]); let bytes = body.len(); - sink.send(body).await.map(|()| (end - begin, bytes)) + sink.send(body).await.map(|()| (rows, bytes)) }); } diff --git a/asap-tools/data-sources/paced-feeder/src/workload.rs b/asap-tools/data-sources/paced-feeder/src/workload.rs index 3b93561e..22e650b0 100644 --- a/asap-tools/data-sources/paced-feeder/src/workload.rs +++ b/asap-tools/data-sources/paced-feeder/src/workload.rs @@ -40,20 +40,21 @@ impl Workload { (format!("g{group}"), format!("i{instance}")) } - /// Samples for sub-second slot `slot` of one-second tick `tick`, one per - /// series. Slots are spaced evenly over `(t, t + 1 s]`, where `t` is the - /// tick's start: the same left-open interval as ASAPQuery's one-second - /// windows, so a tick never spans two of them. - pub fn slot_rows(&self, tick: u64, slot: u32, start_ms: i64) -> Vec { + /// The samples of one-second tick `tick`, series-major: each series' + /// `samples_per_sec` samples in timestamp order, then the next series'. + /// Slot `j` of `n` is at `t + (j + 1) · 1 s / n`, where `t` is the tick's + /// start, so the tick lies in `(t, t + 1 s]`: the same left-open interval as + /// ASAPQuery's one-second windows, so a tick never spans two of them. + pub fn tick_rows(&self, tick: u64, start_ms: i64) -> Vec { let per_sec = u64::from(self.samples_per_sec); - let slot = u64::from(slot); - let timestamp_ms = start_ms + tick as i64 * 1_000 + ((slot + 1) * 1_000 / per_sec) as i64; - let sample_index = tick * per_sec + slot; + let tick_ms = start_ms + tick as i64 * 1_000; (0..self.num_series()) - .map(|series| Row { - series, - timestamp_ms, - value: self.value(series, sample_index), + .flat_map(|series| { + (0..per_sec).map(move |slot| Row { + series, + timestamp_ms: tick_ms + ((slot + 1) * 1_000 / per_sec) as i64, + value: self.value(series, tick * per_sec + slot), + }) }) .collect() } @@ -66,6 +67,23 @@ impl Workload { } } +/// Row ranges of the requests for `rows` series-major rows with +/// `per_series` rows per series: about `batch_rows` each, cut only between +/// series (at least one series per request), so every series' samples of a +/// tick reach the sink in one request, in timestamp order. Concurrent requests +/// then reorder samples only across series, never within one, which keeps +/// counters monotonic for consumers that read them in arrival order. +pub fn series_batches( + rows: usize, + per_series: usize, + batch_rows: usize, +) -> impl Iterator> { + let batch = (batch_rows / per_series).max(1) * per_series; + (0..rows) + .step_by(batch) + .map(move |begin| begin..(begin + batch).min(rows)) +} + /// Counter-based uniform draw in (0, 1]: a stateless hash of the inputs, so /// any sample can be regenerated independently of the ones before it. fn unit_uniform(seed: u64, series: u64, sample_index: u64) -> f64 { @@ -97,37 +115,59 @@ mod tests { } #[test] - fn each_slot_has_every_series_at_its_timestamp() { + fn a_tick_is_series_major_in_timestamp_order() { let w = workload(); - for (slot, timestamp_ms) in [(0, 15_500), (1, 16_000)] { - let rows = w.slot_rows(5, slot, 10_000); - assert_eq!(rows.len(), 6); - assert!(rows.iter().all(|r| r.timestamp_ms == timestamp_ms)); - assert!(rows.iter().map(|r| r.series).eq(0..6)); + let rows = w.tick_rows(5, 10_000); + assert_eq!(rows.len(), 12); + for (series, samples) in rows.chunks(2).enumerate() { + assert!(samples.iter().all(|r| r.series == series)); + let timestamps: Vec = samples.iter().map(|r| r.timestamp_ms).collect(); + assert_eq!(timestamps, [15_500, 16_000]); + } + } + + #[test] + fn batches_keep_each_series_in_one_request_in_order() { + let w = Workload { + samples_per_sec: 3, + ..workload() + }; + let rows = w.tick_rows(2, 0); + // 7 rows round down to 2 series (6 rows); 1 row still takes a series. + for (batch_rows, sizes) in [(7, vec![6, 6, 6]), (1, vec![3; 6]), (100, vec![18])] { + let batches: Vec<_> = series_batches(rows.len(), 3, batch_rows).collect(); + assert_eq!(batches.iter().map(|b| b.len()).collect::>(), sizes); + for series in 0..w.num_series() { + let holding: Vec<_> = batches + .iter() + .filter(|b| rows[(*b).clone()].iter().any(|r| r.series == series)) + .collect(); + assert_eq!(holding.len(), 1, "series {series}"); + let timestamps: Vec = rows[holding[0].clone()] + .iter() + .filter(|r| r.series == series) + .map(|r| r.timestamp_ms) + .collect(); + assert_eq!(timestamps.len(), 3); + assert!(timestamps.windows(2).all(|t| t[0] < t[1])); + } } } #[test] fn values_depend_only_on_seed_series_and_sample_index() { let w = workload(); - // Same slot, different start time: same values. - let a: Vec = w.slot_rows(4, 1, 0).iter().map(|r| r.value).collect(); - let b: Vec = w - .slot_rows(4, 1, 1_000_000) - .iter() - .map(|r| r.value) - .collect(); - assert_eq!(a, b); + let values = |w: &Workload, start_ms| -> Vec { + w.tick_rows(4, start_ms).iter().map(|r| r.value).collect() + }; + // Same tick, different start time: same values. + assert_eq!(values(&w, 0), values(&w, 1_000_000)); // A different seed changes them. - let other = Workload { seed: 8, ..w }; - assert_ne!( - a, - other - .slot_rows(4, 1, 0) - .iter() - .map(|r| r.value) - .collect::>() - ); + let other = Workload { + seed: 8, + ..w.clone() + }; + assert_ne!(values(&w, 0), values(&other, 0)); } #[test] From c5842b57f2f9446427b70bda3a21641a3afc5e60 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Tue, 13 Oct 2026 23:02:37 -0400 Subject: [PATCH 7/8] fix(asap-tools): reject non-whole-second --start-ms in paced feeder A tick straddling a 1s boundary can close a window before its other series' samples arrive. Co-Authored-By: Claude Opus 5.5 --- .../data-sources/paced-feeder/src/main.rs | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/asap-tools/data-sources/paced-feeder/src/main.rs b/asap-tools/data-sources/paced-feeder/src/main.rs index 184d85e7..498067ac 100644 --- a/asap-tools/data-sources/paced-feeder/src/main.rs +++ b/asap-tools/data-sources/paced-feeder/src/main.rs @@ -180,6 +180,15 @@ impl Sink { } } +/// A tick's samples lie in one 1s window only if ticks start on whole seconds; +/// a tick straddling a boundary can close a window before its other series arrive. +fn validate_start_ms(start_ms: i64) -> Result<()> { + if start_ms % 1_000 != 0 { + bail!("--start-ms must be a whole second (multiple of 1000), got {start_ms}"); + } + Ok(()) +} + fn next_whole_second_ms() -> i64 { let now_ms = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -230,6 +239,9 @@ async fn main() -> Result<()> { if batch_rows == 0 || args.concurrency == 0 { bail!("--batch-rows and --concurrency must be positive"); } + if let Some(start_ms) = args.start_ms { + validate_start_ms(start_ms)?; + } let workload = Workload { seed: args.seed, @@ -370,3 +382,14 @@ async fn main() -> Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn start_ms_must_be_whole_second() { + assert!(validate_start_ms(1_700_000_000_000).is_ok()); + assert!(validate_start_ms(1_700_000_000_500).is_err()); + } +} From 14fccbd4e092dc0e4ed7709553e036d6a46015b8 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 09:47:29 -0400 Subject: [PATCH 8/8] feat(asap-tools): let the paced feeder backdate its data with --send-at-ms Feeders launched at different moments can share one aligned event-time origin: data is stamped from --start-ms but sent from --send-at-ms. Co-Authored-By: Claude Opus 5.5 --- .../data-sources/paced-feeder/README.md | 3 +- .../data-sources/paced-feeder/src/main.rs | 33 +++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/asap-tools/data-sources/paced-feeder/README.md b/asap-tools/data-sources/paced-feeder/README.md index ab41bd70..f5c1acbc 100644 --- a/asap-tools/data-sources/paced-feeder/README.md +++ b/asap-tools/data-sources/paced-feeder/README.md @@ -160,7 +160,8 @@ http: 2000000 records/s ``` Event time starts at the next whole wall-clock second (or `--start-ms`), so -`NOW()`-relative queries see the data as it arrives. +`NOW()`-relative queries see the data as it arrives. `--send-at-ms` sends the +first tick at a later wall-clock time, backdating the data by the difference. ## Usage diff --git a/asap-tools/data-sources/paced-feeder/src/main.rs b/asap-tools/data-sources/paced-feeder/src/main.rs index 0e888038..65db1296 100644 --- a/asap-tools/data-sources/paced-feeder/src/main.rs +++ b/asap-tools/data-sources/paced-feeder/src/main.rs @@ -93,6 +93,11 @@ struct Args { /// next whole wall-clock second, so `NOW()`-relative queries see live data. #[arg(long)] start_ms: Option, + /// Wall-clock time to send the first tick, in ms since the epoch. Defaults + /// to `--start-ms`; a later time backdates the data by the difference, so + /// feeders launched at different moments share one event-time origin. + #[arg(long, requires = "start_ms")] + send_at_ms: Option, /// Rows per HTTP request, for gauges and counters rounded down to whole /// series (at least one). Defaults to 50,000 for remote write, which keeps @@ -134,6 +139,7 @@ struct RunStats { samples_per_sec: Option, seed: u64, start_ms: i64, + send_at_ms: i64, duration_secs: u64, target_rows_per_sec: usize, rows_sent: usize, @@ -264,6 +270,14 @@ fn next_whole_second_ms() -> i64 { (now_ms / 1_000 + 1) * 1_000 } +/// Data sent before its own timestamp would be in the future. +fn check_send_at(start_ms: i64, send_at_ms: i64) -> Result<()> { + if send_at_ms < start_ms { + bail!("--send-at-ms ({send_at_ms}) must not be before --start-ms ({start_ms})"); + } + Ok(()) +} + /// Sleep until wall-clock `epoch_ms`, then return the monotonic instant it maps to. async fn wait_for_wall_clock(epoch_ms: i64) -> Instant { let now_ms = SystemTime::now() @@ -426,6 +440,8 @@ async fn main() -> Result<()> { let sink = Arc::new(Sink::new(&args, sink_kind, url, &workload, batch_rows)?); let permits = Arc::new(Semaphore::new(args.concurrency)); let start_ms = args.start_ms.unwrap_or_else(next_whole_second_ms); + let send_at_ms = args.send_at_ms.unwrap_or(start_ms); + check_send_at(start_ms, send_at_ms)?; let metric_names = workload.names(); let series = if workload.metrics.is_empty() { @@ -438,12 +454,12 @@ async fn main() -> Result<()> { ) }; eprintln!( - "paced_feeder: {sink_kind:?} -> {url}, {}{series} = {} rows/s for {duration_secs}s, start_ms={start_ms}", + "paced_feeder: {sink_kind:?} -> {url}, {}{series} = {} rows/s for {duration_secs}s, start_ms={start_ms}, send_at_ms={send_at_ms}", metric_names.join(", "), workload.rows_per_tick(), ); - let base = wait_for_wall_clock(start_ms).await; + let base = wait_for_wall_clock(send_at_ms).await; let mut ticks = Vec::with_capacity(duration_secs as usize); let mut errors: Vec = Vec::new(); let mut failed_requests = 0; @@ -536,6 +552,7 @@ async fn main() -> Result<()> { samples_per_sec: (!workload.metrics.is_empty()).then_some(workload.samples_per_sec), seed: args.seed, start_ms, + send_at_ms, duration_secs, target_rows_per_sec: workload.rows_per_tick(), rows_sent, @@ -570,3 +587,15 @@ async fn main() -> Result<()> { } Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn send_at_may_backdate_but_not_postdate_the_data() { + assert!(check_send_at(1_000, 1_000).is_ok()); + assert!(check_send_at(1_000, 301_000).is_ok()); + assert!(check_send_at(1_000, 999).is_err()); + } +}