From be66ac5211e32d71c6161fdc3c3bb0f4704cc380 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Sun, 11 Oct 2026 20:11:33 -0400 Subject: [PATCH 1/9] feat(planner): asap-planner --planner saved writes a sketch-bench eval plan without re-solving Reads a plan from sketch-bench's autosketch_vs_asap plan file and the eval table's `e2e` block (PromQL queries, the plan RQEs each reads, metric labels), and writes streaming and inference YAML through the same converter as --planner milp. experiment_run_e2e.py gains controller.prebuilt_output_dir to deploy those configs instead of running the planner. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + asap-planner-rs/Cargo.toml | 1 + asap-planner-rs/src/main.rs | 96 ++++- .../src/optimizer/aqe_extractor.rs | 2 +- asap-planner-rs/src/optimizer/milp.rs | 13 +- asap-planner-rs/src/optimizer/milp_output.rs | 42 +- asap-planner-rs/src/optimizer/mod.rs | 11 +- asap-planner-rs/src/optimizer/saved_plan.rs | 407 ++++++++++++++++++ asap-tools/experiments/config/config.yaml | 3 + asap-tools/experiments/experiment_run_e2e.py | 46 +- .../experiments/experiment_utils/config.py | 12 + 11 files changed, 600 insertions(+), 34 deletions(-) create mode 100644 asap-planner-rs/src/optimizer/saved_plan.rs diff --git a/Cargo.lock b/Cargo.lock index e1d15191..50c54223 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -166,6 +166,7 @@ dependencies = [ "chrono", "clap 4.6.1", "elastic_dsl_utilities", + "flate2", "indexmap 2.14.0", "pretty_assertions", "promql-parser", diff --git a/asap-planner-rs/Cargo.toml b/asap-planner-rs/Cargo.toml index 5e9bf33d..0c89320d 100644 --- a/asap-planner-rs/Cargo.toml +++ b/asap-planner-rs/Cargo.toml @@ -37,6 +37,7 @@ indexmap.workspace = true chrono.workspace = true promql-parser = "0.5.0" reqwest = { version = "0.11", features = ["blocking", "json"] } +flate2 = "1.1" rqe-optimizer = { git = "https://github.com/ProjectASAP/sketch-bench", branch = "main" } [dev-dependencies] diff --git a/asap-planner-rs/src/main.rs b/asap-planner-rs/src/main.rs index c4b2e20b..e4948501 100644 --- a/asap-planner-rs/src/main.rs +++ b/asap-planner-rs/src/main.rs @@ -1,5 +1,6 @@ use asap_planner::optimizer::{ - parse_weight, plan_milp, plan_to_planner_output, reject_unwritable_queries, MilpInputs, + parse_weight, plan_milp, plan_to_planner_output, reject_unwritable_queries, saved_plan_output, + MilpInputs, PlanSelector, }; use asap_planner::{ Controller, ControllerConfig, ElasticController, ElasticRuntimeOptions, RuntimeOptions, @@ -57,10 +58,33 @@ struct Args { clickhouse_database: Option, /// `milp` plans with sketch-bench's rqe-optimizer: PromQL with - /// --input_config only, labels from its `metrics:` hints. + /// --input_config only, labels from its `metrics:` hints. `saved` writes + /// a plan sketch-bench's `autosketch_vs_asap` already solved. #[arg(long, value_enum, default_value = "legacy")] planner: PlannerArg, + /// Saved only. sketch-bench `autosketch_vs_asap` plan file + /// (`.plans.json.gz`). + #[arg(long, required_if_eq("planner", "saved"))] + plan: Option, + + /// Saved only. The eval workload table the plan was solved for; its + /// `e2e` block gives the PromQL queries and metric labels. + #[arg(long = "workload-table", required_if_eq("planner", "saved"))] + workload_table: Option, + + /// Saved only. The plan's method, e.g. `asap`. + #[arg(long, required_if_eq("planner", "saved"))] + method: Option, + + /// Saved only. The plan's objective weights, e.g. `cpu` or `fargate`. + #[arg(long, required_if_eq("planner", "saved"))] + weights: Option, + + /// Saved only. The plan at this latency SLA; without it, the cheapest. + #[arg(long = "sla-ms")] + sla_ms: Option, + /// MILP only. YAML workload facts: per metric, positive `value_range` and /// `cardinality` per label set, including all labels (the series count), /// plus the `shape` of each grouping sketches may serve. @@ -99,6 +123,7 @@ enum EngineArg { enum PlannerArg { Legacy, Milp, + Saved, } fn main() -> anyhow::Result<()> { @@ -116,8 +141,22 @@ fn main() -> anyhow::Result<()> { EngineArg::Precompute => StreamingEngine::Precompute, }; - if args.planner == PlannerArg::Milp { - run_milp(&args)?; + if args.planner != PlannerArg::Saved { + anyhow::ensure!( + args.plan.is_none() + && args.workload_table.is_none() + && args.method.is_none() + && args.weights.is_none() + && args.sla_ms.is_none(), + "--plan, --workload-table, --method, --weights and --sla-ms require --planner saved" + ); + } + if args.planner != PlannerArg::Legacy { + match args.planner { + PlannerArg::Milp => run_milp(&args)?, + PlannerArg::Saved => run_saved(&args)?, + PlannerArg::Legacy => unreachable!(), + } println!("Generated configs in {}", args.output_dir.display()); return Ok(()); } @@ -274,3 +313,52 @@ fn run_milp(args: &Args) -> anyhow::Result<()> { .write_to_dir(&args.output_dir)?; Ok(()) } + +fn run_saved(args: &Args) -> anyhow::Result<()> { + anyhow::ensure!( + args.input_config.is_none() && args.query_log.is_none(), + "--planner saved reads its queries from --workload-table, not --input_config or --query-log" + ); + anyhow::ensure!( + matches!(args.query_language, QueryLanguage::promql) + && args.prometheus_url.is_none() + && !args.enable_punting + && args.range_duration_ms == 0 + && args.step_ms == 0 + && args.clickhouse_url.is_none() + && args.clickhouse_database.is_none(), + "--planner saved takes only PromQL and none of --prometheus-url, --enable-punting, \ + --range-duration-ms, --step-ms, --clickhouse-url, --clickhouse-database" + ); + anyhow::ensure!( + args.workload_facts.is_none() + && args.atomic_costs.is_none() + && args.saturation_dir.is_none() + && args.w_cpu.is_none() + && args.w_mem.is_none(), + "--workload-facts, --atomic-costs, --saturation-dir, --w-cpu and --w-mem require \ + --planner milp" + ); + let scrape_interval_ms = args + .data_ingestion_interval_ms + .filter(|&ms| ms > 0) + .ok_or_else(|| { + anyhow::anyhow!("--planner saved requires a positive --data-ingestion-interval-ms") + })?; + saved_plan_output( + args.workload_table + .as_deref() + .expect("clap requires --workload-table with --planner saved"), + args.plan + .as_deref() + .expect("clap requires --plan with --planner saved"), + &PlanSelector { + method: args.method.as_deref().expect("clap requires --method"), + weights: args.weights.as_deref().expect("clap requires --weights"), + sla_ms: args.sla_ms, + }, + scrape_interval_ms, + )? + .write_to_dir(&args.output_dir)?; + Ok(()) +} diff --git a/asap-planner-rs/src/optimizer/aqe_extractor.rs b/asap-planner-rs/src/optimizer/aqe_extractor.rs index 1a7b3262..00ff2415 100644 --- a/asap-planner-rs/src/optimizer/aqe_extractor.rs +++ b/asap-planner-rs/src/optimizer/aqe_extractor.rs @@ -132,7 +132,7 @@ fn normalized_f64_bits(value: f64) -> u64 { /// dropped — they contribute no AQE. Only arithmetic operators are split; /// comparison and set operators are left as-is (treated as opaque leaves). /// An `avg` leaf becomes its sum and count leaves (see `rewrite_avg`). -fn decompose_to_leaves(query: &str) -> Vec { +pub(crate) fn decompose_to_leaves(query: &str) -> Vec { let (lhs, rhs) = match parse_binary_arms(query) { Some(arms) => arms, None => return rewrite_avg(query), diff --git a/asap-planner-rs/src/optimizer/milp.rs b/asap-planner-rs/src/optimizer/milp.rs index 71d8d83c..2f53fca9 100644 --- a/asap-planner-rs/src/optimizer/milp.rs +++ b/asap-planner-rs/src/optimizer/milp.rs @@ -243,6 +243,17 @@ pub fn build_milp_workload( config: &ControllerConfig, facts: &WorkloadFacts, scrape_interval_ms: u64, +) -> Result { + let workload = workload_from_config(config, scrape_interval_ms)?; + validate_facts(&workload.raqes, facts).map_err(MilpError::InvalidInputs)?; + Ok(workload) +} + +/// [`build_milp_workload`] without the facts check, for a plan solved +/// elsewhere. +pub fn workload_from_config( + config: &ControllerConfig, + scrape_interval_ms: u64, ) -> Result { // The MILP chooses windows and sketch parameters itself. let unsupported: Vec<&'static str> = [ @@ -298,10 +309,8 @@ pub fn build_milp_workload( tracing::debug!( items = items.len(), raqes = raqes.len(), - metrics_with_facts = facts.len(), "milp inputs: built raqes" ); - validate_facts(&raqes, facts).map_err(MilpError::InvalidInputs)?; Ok(MilpWorkload { items, raqes, diff --git a/asap-planner-rs/src/optimizer/milp_output.rs b/asap-planner-rs/src/optimizer/milp_output.rs index 8f934931..f822edb0 100644 --- a/asap-planner-rs/src/optimizer/milp_output.rs +++ b/asap-planner-rs/src/optimizer/milp_output.rs @@ -92,14 +92,36 @@ pub fn plan_to_planner_output( workload: &MilpWorkload, solution: &MilpSolution, ) -> Result { + let deployments: Vec = solution + .deployments + .iter() + .map(|planned| planned.deployment.clone()) + .collect(); + let served_by: Vec = solution.raqes.iter().map(|r| r.deployment).collect(); + deployments_to_planner_output(config, workload, &deployments, &served_by) +} + +/// [`plan_to_planner_output`] for a plan given as its deployments and, per +/// Raqe of `workload`, the index of the deployment serving it. +pub fn deployments_to_planner_output( + config: &ControllerConfig, + workload: &MilpWorkload, + deployments: &[Deployment], + served_by: &[usize], +) -> Result { + assert_eq!( + served_by.len(), + workload.raqes.len(), + "one deployment per Raqe" + ); reject_unwritable_queries(config)?; // sketch-bench lets a deployment serve a RAQE grouped by a subset of its // grouping (a roll-up, sketch-bench#190). The engine reads a deployment // only at its own grouping, and `aggregation_config` takes the labels // from the first RAQE served, so refuse a roll-up rather than write // configs at the wrong grouping. - for (raqe, planned) in workload.raqes.iter().zip(&solution.raqes) { - let deployment = &solution.deployments[planned.deployment].deployment; + for (raqe, &index) in workload.raqes.iter().zip(served_by) { + let deployment = &deployments[index]; if raqe.grouping_labels != deployment.grouping_labels { return Err(MilpOutputError::RollupUnsupported { query: raqe.id.clone(), @@ -111,26 +133,26 @@ pub fn plan_to_planner_output( let item_of = |raqe: usize| &workload.items[workload.raqe_items[raqe]]; // Each item once per deployment, however many occurrences it has. - let mut served: Vec> = vec![Vec::new(); solution.deployments.len()]; + let mut served: Vec> = vec![Vec::new(); deployments.len()]; let mut seen = HashSet::new(); - for (raqe, planned) in solution.raqes.iter().enumerate() { - if seen.insert((planned.deployment, workload.raqe_items[raqe])) { - served[planned.deployment].push(item_of(raqe)); + for (raqe, &index) in served_by.iter().enumerate() { + if seen.insert((index, workload.raqe_items[raqe])) { + served[index].push(item_of(raqe)); } } let mut aggregations: IndexMap = IndexMap::new(); let mut deployment_keys = Vec::with_capacity(served.len()); - for (planned, items) in solution.deployments.iter().zip(&served) { - let aggregation = aggregation_config(&planned.deployment, items)?; + for (deployment, items) in deployments.iter().zip(&served) { + let aggregation = aggregation_config(deployment, items)?; let key = aggregation.identifying_key(); aggregations.entry(key.clone()).or_insert(aggregation); deployment_keys.push(key); } let mut queries: IndexMap = IndexMap::new(); - for (raqe, planned) in solution.raqes.iter().enumerate() { - let key = &deployment_keys[planned.deployment]; + for (raqe, &index) in served_by.iter().enumerate() { + let key = &deployment_keys[index]; for query in &item_of(raqe).query_strings { match queries.entry(query.clone()) { Entry::Vacant(entry) => { diff --git a/asap-planner-rs/src/optimizer/mod.rs b/asap-planner-rs/src/optimizer/mod.rs index 7731c14c..2b0b46cc 100644 --- a/asap-planner-rs/src/optimizer/mod.rs +++ b/asap-planner-rs/src/optimizer/mod.rs @@ -3,6 +3,7 @@ pub mod atomic_costs; pub mod error; pub mod milp; pub mod milp_output; +pub mod saved_plan; pub mod solution; pub mod workload_facts; @@ -10,9 +11,13 @@ pub use aqe_extractor::{extract_aqes, RQE}; pub use atomic_costs::{load_flat_atomic_cost_table, AtomicCostEntry, AtomicCostTable}; pub use error::OptimizerError; pub use milp::{ - build_milp_workload, parse_weight, plan_milp, solve_milp, MilpError, MilpInputs, MilpPlan, - MilpWorkload, + build_milp_workload, parse_weight, plan_milp, solve_milp, workload_from_config, MilpError, + MilpInputs, MilpPlan, MilpWorkload, }; -pub use milp_output::{plan_to_planner_output, reject_unwritable_queries, MilpOutputError}; +pub use milp_output::{ + deployments_to_planner_output, plan_to_planner_output, reject_unwritable_queries, + MilpOutputError, +}; +pub use saved_plan::{saved_plan_output, PlanSelector}; pub use solution::OptimizerItem; pub use workload_facts::{load_workload_facts, parse_workload_facts, WorkloadFactsError}; diff --git a/asap-planner-rs/src/optimizer/saved_plan.rs b/asap-planner-rs/src/optimizer/saved_plan.rs new file mode 100644 index 00000000..02ceecab --- /dev/null +++ b/asap-planner-rs/src/optimizer/saved_plan.rs @@ -0,0 +1,407 @@ +//! Writes a plan solved elsewhere, sketch-bench `autosketch_vs_asap`'s plan +//! file, as the planner's streaming and inference YAML, without re-solving. +//! The workload comes from the eval table's `e2e` block: its PromQL queries, +//! the plan RQEs each reads, and the metrics' labels. + +use std::collections::{BTreeMap, HashMap}; +use std::io::Read; +use std::path::Path; + +use anyhow::{bail, ensure, Context}; +use rqe_optimizer::{AtomicCostEntry, Deployment}; +use serde::Deserialize; +use serde_json::{json, Value}; + +use crate::config::input::ControllerConfig; +use crate::planner_output::PlannerOutput; + +use super::aqe_extractor::decompose_to_leaves; +use super::milp::workload_from_config; +use super::milp_output::deployments_to_planner_output; + +#[derive(Deserialize)] +struct Table { + workloads: Vec, +} + +#[derive(Deserialize)] +struct TableWorkload { + e2e: Option, +} + +#[derive(Deserialize)] +struct E2e { + queries: Vec, + metrics: BTreeMap, +} + +#[derive(Deserialize)] +struct E2eQuery { + promql: String, + interval_secs: u64, + /// Plan RQE ids, one per leaf of `promql` in order. + rqes: Vec, +} + +#[derive(Deserialize)] +struct E2eMetric { + labels: Vec, +} + +#[derive(Deserialize)] +struct E2eLabel { + name: String, +} + +#[derive(Deserialize)] +struct PlanFile { + plans: Vec, +} + +#[derive(Deserialize)] +struct Plan { + method: String, + weights: String, + kind: String, + sla_ms: Option, + chosen: Vec, +} + +#[derive(Deserialize)] +struct Chosen { + rqe: String, + sketch: String, + params: Value, + window_secs: u64, + slide_secs: u64, + /// Index of the plan's deployment. + deployment: usize, + /// Present only when the deployment's grouping differs from the RQE's. + deployment_grouping: Option, +} + +/// Which plan of the file to write. +pub struct PlanSelector<'a> { + pub method: &'a str, + pub weights: &'a str, + /// `None` for the cheapest plan, else the plan at this latency SLA. + pub sla_ms: Option, +} + +/// The controller config the table's `e2e` block describes: one query group +/// per repeat interval. +fn e2e_config(e2e: &E2e) -> anyhow::Result { + let mut by_interval: BTreeMap> = BTreeMap::new(); + for q in &e2e.queries { + by_interval + .entry(q.interval_secs) + .or_default() + .push(&q.promql); + } + let config = json!({ + "query_groups": by_interval.iter().map(|(secs, queries)| json!({ + "queries": queries, + "repetition_delay_ms": secs * 1000, + })).collect::>(), + "metrics": e2e.metrics.iter().map(|(metric, m)| json!({ + "metric": metric, + "labels": m.labels.iter().map(|l| &l.name).collect::>(), + })).collect::>(), + }); + Ok(serde_json::from_value(config)?) +} + +fn read_json(path: &Path) -> anyhow::Result { + let bytes = std::fs::read(path).with_context(|| format!("reading {}", path.display()))?; + let text = if bytes.starts_with(&[0x1f, 0x8b]) { + let mut text = String::new(); + flate2::read::GzDecoder::new(&bytes[..]).read_to_string(&mut text)?; + text + } else { + String::from_utf8(bytes)? + }; + serde_json::from_str(&text).with_context(|| format!("parsing {}", path.display())) +} + +/// Streaming and inference YAML for the selected plan of `plan_path` (JSON, +/// gzipped or not) over the workload of `table_path`. +pub fn saved_plan_output( + table_path: &Path, + plan_path: &Path, + selector: &PlanSelector, + scrape_interval_ms: u64, +) -> anyhow::Result { + let table: Table = read_json(table_path)?; + let [TableWorkload { e2e: Some(e2e) }] = table.workloads.as_slice() else { + bail!( + "{}: expected one workload with an `e2e` block", + table_path.display() + ); + }; + let plans: PlanFile = read_json(plan_path)?; + let kind = if selector.sla_ms.is_some() { + "sla" + } else { + "cheapest" + }; + let matching: Vec<&Plan> = plans + .plans + .iter() + .filter(|p| { + p.method == selector.method + && p.weights == selector.weights + && p.kind == kind + && p.sla_ms == selector.sla_ms + }) + .collect(); + let [plan] = matching.as_slice() else { + bail!( + "{}: {} plans match method {:?}, weights {:?}, {kind} {:?}; expected one", + plan_path.display(), + matching.len(), + selector.method, + selector.weights, + selector.sla_ms, + ); + }; + let chosen: HashMap<&str, &Chosen> = plan.chosen.iter().map(|c| (c.rqe.as_str(), c)).collect(); + + // Leaf query string and repeat interval -> the plan RQE it reads. + let mut rqe_of: HashMap<(String, u64), &str> = HashMap::new(); + for q in &e2e.queries { + let leaves = decompose_to_leaves(&q.promql); + ensure!( + leaves.len() == q.rqes.len(), + "query {:?} has {} leaves but names {} RQEs", + q.promql, + leaves.len(), + q.rqes.len() + ); + for (leaf, rqe) in leaves.into_iter().zip(&q.rqes) { + let key = (leaf, q.interval_secs * 1000); + if let Some(other) = rqe_of.insert(key.clone(), rqe) { + ensure!( + other == rqe, + "leaf {:?} every {} ms reads both {other:?} and {rqe:?}", + key.0, + key.1 + ); + } + } + } + + let config = e2e_config(e2e)?; + let workload = workload_from_config(&config, scrape_interval_ms)?; + + // Plan deployment index -> its position in `deployments`, in order of + // the first Raqe each serves. + let mut position: HashMap = HashMap::new(); + let mut deployments: Vec = Vec::new(); + let mut served_by = Vec::with_capacity(workload.raqes.len()); + for (raqe, &item) in workload.raqes.iter().zip(&workload.raqe_items) { + let item = &workload.items[item]; + // An item's leaves (say p90 and p50 over one window) read one + // deployment, so their picks must name the same sketch. + let mut index = None; + for leaf in &item.query_strings { + let rqe = rqe_of + .get(&(leaf.clone(), item.t_repeat_ms)) + .with_context(|| format!("leaf {leaf:?} has no RQE in the table's e2e block"))?; + let pick = chosen + .get(rqe) + .with_context(|| format!("the plan doesn't serve RQE {rqe:?}"))?; + ensure!( + pick.deployment_grouping.is_none(), + "RQE {rqe:?} is served by a roll-up, which the engine can't read yet" + ); + let deployment = Deployment { + capability: raqe.capability, + metric: raqe.metric.clone(), + spatial_filter: raqe.spatial_filter.clone(), + grouping_labels: raqe.grouping_labels.clone(), + config: cost_entry(pick)?, + window_ms: pick.window_secs * 1000, + slide_ms: pick.slide_secs * 1000, + key_tracker: None, + }; + let at = *position.entry(pick.deployment).or_insert_with(|| { + deployments.push(deployment.clone()); + deployments.len() - 1 + }); + // One plan deployment is one sketch: every Raqe it serves must agree. + ensure!( + same_sketch(&deployments[at], &deployment), + "plan deployment {} serves Raqes that need different sketches: {:?} and {deployment:?}", + pick.deployment, + deployments[at] + ); + ensure!( + index.is_none_or(|index| index == at), + "query {:?} is served by two deployments", + item.query_strings + ); + index = Some(at); + } + let index = index.expect("an item has a query string"); + served_by.push(index); + } + Ok(deployments_to_planner_output( + &config, + &workload, + &deployments, + &served_by, + )?) +} + +fn same_sketch(a: &Deployment, b: &Deployment) -> bool { + ( + a.capability, + &a.metric, + &a.spatial_filter, + &a.grouping_labels, + &a.config.sketch, + &a.config.sketch_config, + a.window_ms, + a.slide_ms, + ) == ( + b.capability, + &b.metric, + &b.spatial_filter, + &b.grouping_labels, + &b.config.sketch, + &b.config.sketch_config, + b.window_ms, + b.slide_ms, + ) +} + +/// The sketch config the YAML is written from. Costs and accuracy don't +/// reach the YAML, so they are zero. +fn cost_entry(pick: &Chosen) -> anyhow::Result { + Ok(serde_json::from_value(json!({ + "sketch": pick.sketch, + "sketch_config": {"algorithm": pick.sketch, "params": pick.params}, + "mem_bytes_per_instance": 0.0, + "insert_cpu_secs": 0.0, + "merge_cpu_secs": 0.0, + "query_cpu_secs": 0.0, + "query_accuracy": {}, + "accuracy_metric": "", + "measured_at": { + "items_per_instance": 0, + "keys_per_instance": null, + "value_range": null, + "merge_operand_items": null, + "distribution": null, + }, + }))?) +} + +#[cfg(test)] +mod tests { + use super::*; + + const TABLE: &str = r#"{"workloads": [{"e2e": { + "queries": [ + {"promql": "sum by (job) (data)", "interval_secs": 1, "rqes": ["sum"]}, + {"promql": "quantile_over_time(0.9, data[5m]) / quantile_over_time(0.5, data[5m])", + "interval_secs": 60, "rqes": ["p90", "p50"]}, + {"promql": "quantile_over_time(0.5, data[5m])", "interval_secs": 60, "rqes": ["p50"]} + ], + "metrics": {"data": {"type": "gauge", "labels": [ + {"name": "job", "cardinality": 10}, + {"name": "label_0", "cardinality": 1000, "child_of": "job"} + ]}} + }}]}"#; + + fn pick(rqe: &str, deployment: usize, sketch: &str, params: Value, window: u64) -> Value { + json!({"rqe": rqe, "deployment": deployment, "sketch": sketch, "params": params, + "window_secs": window, "slide_secs": 60.min(window)}) + } + + /// The cheapest `asap`/`cpu` plan: the ratio's legs share one sliding + /// DDSketch. The SLA plan differs only in alpha. + fn plans(p50: Value) -> Value { + let dd = |alpha: f64| json!({"alpha": alpha}); + json!({"plans": [ + {"method": "asap", "weights": "cpu", "kind": "cheapest", "chosen": [ + pick("sum", 3, "exact-sum", json!({}), 1), + pick("p90", 0, "dd", dd(0.02), 300), + p50, + ]}, + {"method": "asap", "weights": "cpu", "kind": "sla", "sla_ms": 100.0, "chosen": [ + pick("sum", 3, "exact-sum", json!({}), 1), + pick("p90", 0, "dd", dd(0.01), 300), + pick("p50", 0, "dd", dd(0.01), 300), + ]}, + ]}) + } + + fn run(plan: Value, sla_ms: Option) -> anyhow::Result { + let dir = tempfile::tempdir().unwrap(); + let (table_path, plan_path) = (dir.path().join("table.json"), dir.path().join("plan.json")); + std::fs::write(&table_path, TABLE).unwrap(); + std::fs::write(&plan_path, plan.to_string()).unwrap(); + let selector = PlanSelector { + method: "asap", + weights: "cpu", + sla_ms, + }; + saved_plan_output(&table_path, &plan_path, &selector, 1000) + } + + fn refusal(plan: Value, sla_ms: Option) -> String { + run(plan, sla_ms).err().expect("refused").to_string() + } + + fn shared_p50() -> Value { + pick("p50", 0, "dd", json!({"alpha": 0.02}), 300) + } + + #[test] + fn writes_one_aggregation_per_plan_deployment() { + let output = run(plans(shared_p50()), None).unwrap(); + assert_eq!(output.streaming_aggregation_count(), 2); + // The ratio's two leaves and the sum; the standalone p50 is a leaf already. + assert_eq!(output.inference_query_count(), 3); + let yaml = output.to_streaming_yaml_string().unwrap(); + assert!(yaml.contains("windowSizeMs: 300000"), "{yaml}"); + assert!(yaml.contains("slideIntervalMs: 60000"), "{yaml}"); + assert!(yaml.contains("windowType: sliding"), "{yaml}"); + assert!(yaml.contains("alpha: 0.02"), "{yaml}"); + } + + #[test] + fn sla_ms_selects_the_plan_at_that_sla() { + let yaml = run(plans(shared_p50()), Some(100.0)) + .unwrap() + .to_streaming_yaml_string() + .unwrap(); + assert!(yaml.contains("alpha: 0.01"), "{yaml}"); + let err = refusal(plans(shared_p50()), Some(300.0)); + assert!(err.contains("0 plans match"), "{err}"); + } + + #[test] + fn refuses_a_roll_up() { + let mut p50 = shared_p50(); + p50["deployment_grouping"] = json!(["job"]); + let err = refusal(plans(p50), None); + assert!(err.contains("roll-up"), "{err}"); + } + + #[test] + fn refuses_one_deployment_with_two_sketches() { + let p50 = pick("p50", 0, "dd", json!({"alpha": 0.05}), 300); + let err = refusal(plans(p50), None); + assert!(err.contains("different sketches"), "{err}"); + } + + #[test] + fn refuses_a_plan_missing_an_rqe() { + let mut plan = plans(shared_p50()); + plan["plans"][0]["chosen"].as_array_mut().unwrap().pop(); + let err = refusal(plan, None); + assert!(err.contains(r#"doesn't serve RQE "p50""#), "{err}"); + } +} diff --git a/asap-tools/experiments/config/config.yaml b/asap-tools/experiments/config/config.yaml index dfd4abd9..8539b4ff 100644 --- a/asap-tools/experiments/config/config.yaml +++ b/asap-tools/experiments/config/config.yaml @@ -90,6 +90,9 @@ query_engine: controller: punting: ??? # REQUIRED: Enable query punting based on performance heuristics (should_be_performant check) auto_discover_labels: false # Query Prometheus for metric labels instead of using config hints + # Skip the planner and deploy the streaming_config.yaml and inference_config.yaml + # already in this directory, e.g. from `asap-planner --planner saved`. + prebuilt_output_dir: null # Optional manual planner windowing override. Omit this block to preserve the # default tumbling-window behavior. For tumbling windows, omit diff --git a/asap-tools/experiments/experiment_run_e2e.py b/asap-tools/experiments/experiment_run_e2e.py index 9b288ad8..e7188f5b 100644 --- a/asap-tools/experiments/experiment_run_e2e.py +++ b/asap-tools/experiments/experiment_run_e2e.py @@ -1,5 +1,6 @@ import os import json +import shutil import time import hydra @@ -316,20 +317,37 @@ def main(cfg: DictConfig): "Prometheus label discovery is disabled." ) - controller_service.start( - controller_input_file=controller_input_config, - streaming_engine=args.streaming_engine, - controller_remote_output_dir=CONTROLLER_REMOTE_OUTPUT_DIR, - punting=args.controller_punting, - discovery_backend=discovery_backend, - data_ingestion_interval_ms=data_ingestion_interval_ms, - ) - sync.rsync_controller_config_remote_to_local( - provider, - CONTROLLER_REMOTE_OUTPUT_DIR, - CONTROLLER_LOCAL_OUTPUT_DIR, - node_offset=args.node_offset, - ) + if args.controller_prebuilt_output_dir: + print( + f"Skipping the planner: using configs from " + f"{args.controller_prebuilt_output_dir}" + ) + shutil.copytree( + args.controller_prebuilt_output_dir, + CONTROLLER_LOCAL_OUTPUT_DIR, + dirs_exist_ok=True, + ) + sync.rsync_streaming_configs( + provider, + CONTROLLER_LOCAL_OUTPUT_DIR, + CONTROLLER_REMOTE_OUTPUT_DIR, + node_offset=args.node_offset, + ) + else: + controller_service.start( + controller_input_file=controller_input_config, + streaming_engine=args.streaming_engine, + controller_remote_output_dir=CONTROLLER_REMOTE_OUTPUT_DIR, + punting=args.controller_punting, + discovery_backend=discovery_backend, + data_ingestion_interval_ms=data_ingestion_interval_ms, + ) + sync.rsync_controller_config_remote_to_local( + provider, + CONTROLLER_REMOTE_OUTPUT_DIR, + CONTROLLER_LOCAL_OUTPUT_DIR, + node_offset=args.node_offset, + ) if ( config.check_exporter_and_queries_exist( diff --git a/asap-tools/experiments/experiment_utils/config.py b/asap-tools/experiments/experiment_utils/config.py index a731c005..a1a363aa 100644 --- a/asap-tools/experiments/experiment_utils/config.py +++ b/asap-tools/experiments/experiment_utils/config.py @@ -9,6 +9,7 @@ import yaml from typing import Any, Dict, List, Tuple +import hydra.utils from omegaconf import DictConfig, ListConfig, OmegaConf import constants @@ -605,6 +606,17 @@ def __init__(self, cfg: DictConfig): # Controller configuration self.controller_punting = cfg.controller.punting self.controller_auto_discover_labels = cfg.controller.auto_discover_labels + self.controller_prebuilt_output_dir = cfg.controller.get("prebuilt_output_dir") + if self.controller_prebuilt_output_dir: + self.controller_prebuilt_output_dir = hydra.utils.to_absolute_path( + self.controller_prebuilt_output_dir + ) + for name in ("streaming_config.yaml", "inference_config.yaml"): + path = os.path.join(self.controller_prebuilt_output_dir, name) + if not os.path.isfile(path): + raise ValueError( + f"controller.prebuilt_output_dir has no {name}: {path}" + ) # Aggregate cleanup configuration # Valid policies: "circular_buffer", "read_based", "no_cleanup" From e7a49eb8f5b965d0c98186f413f8c6852e9d108e Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Mon, 12 Oct 2026 18:05:37 -0400 Subject: [PATCH 2/9] fix(planner): address review on --planner saved Check saved picks against the RAQE's window rules, refuse duplicate picks, reject avg queries before the leaf split, and take a roll-up's grouping from the plan so the shared guard catches it. Share the flag checks of --planner milp and saved. Copy only the two prebuilt configs into a cleared output dir, refusing the dir itself. Co-Authored-By: Claude Opus 5.5 --- asap-planner-rs/src/main.rs | 83 +++++++---------- asap-planner-rs/src/optimizer/saved_plan.rs | 91 ++++++++++++++++--- asap-tools/experiments/experiment_run_e2e.py | 7 +- .../experiments/experiment_utils/config.py | 31 +++++-- .../tests/test_prebuilt_controller_output.py | 45 +++++++++ 5 files changed, 180 insertions(+), 77 deletions(-) create mode 100644 asap-tools/experiments/tests/test_prebuilt_controller_output.py diff --git a/asap-planner-rs/src/main.rs b/asap-planner-rs/src/main.rs index e4948501..01f556da 100644 --- a/asap-planner-rs/src/main.rs +++ b/asap-planner-rs/src/main.rs @@ -151,6 +151,17 @@ fn main() -> anyhow::Result<()> { "--plan, --workload-table, --method, --weights and --sla-ms require --planner saved" ); } + if args.planner != PlannerArg::Milp { + anyhow::ensure!( + args.workload_facts.is_none() + && args.atomic_costs.is_none() + && args.saturation_dir.is_none() + && args.w_cpu.is_none() + && args.w_mem.is_none(), + "--workload-facts, --atomic-costs, --saturation-dir, --w-cpu and --w-mem require \ + --planner milp" + ); + } if args.planner != PlannerArg::Legacy { match args.planner { PlannerArg::Milp => run_milp(&args)?, @@ -160,15 +171,6 @@ fn main() -> anyhow::Result<()> { println!("Generated configs in {}", args.output_dir.display()); return Ok(()); } - anyhow::ensure!( - args.workload_facts.is_none() - && args.atomic_costs.is_none() - && args.saturation_dir.is_none() - && args.w_cpu.is_none() - && args.w_mem.is_none(), - "--workload-facts, --atomic-costs, --saturation-dir, --w-cpu and --w-mem require \ - --planner milp" - ); match args.query_language { QueryLanguage::promql => { @@ -252,39 +254,43 @@ fn main() -> anyhow::Result<()> { Ok(()) } -fn run_milp(args: &Args) -> anyhow::Result<()> { +/// Checks the flags `--planner milp` and `--planner saved` share: PromQL +/// with labels from `metrics:` hints, no legacy-only flags. Returns the +/// positive scrape interval, which per-series sample rates divide by. +fn check_optimizer_flags(args: &Args, planner: &str) -> anyhow::Result { anyhow::ensure!( matches!(args.query_language, QueryLanguage::promql), - "--planner milp supports only --query-language promql" + "--planner {planner} supports only --query-language promql" ); anyhow::ensure!( args.query_log.is_none(), - "--planner milp needs --input_config: query logs have no `metrics:` hints" + "--planner {planner} doesn't take --query-log: query logs have no `metrics:` hints" ); anyhow::ensure!( args.prometheus_url.is_none(), - "--planner milp takes labels from the config's `metrics:` hints, not --prometheus-url" + "--planner {planner} takes labels from `metrics:` hints, not --prometheus-url" ); anyhow::ensure!( !args.enable_punting && args.range_duration_ms == 0 && args.step_ms == 0, - "--enable-punting, --range-duration-ms and --step-ms don't apply to --planner milp" + "--enable-punting, --range-duration-ms and --step-ms don't apply to --planner {planner}" ); anyhow::ensure!( args.clickhouse_url.is_none() && args.clickhouse_database.is_none(), - "--clickhouse-url and --clickhouse-database don't apply to --planner milp" + "--clickhouse-url and --clickhouse-database don't apply to --planner {planner}" ); + args.data_ingestion_interval_ms + .filter(|&ms| ms > 0) + .ok_or_else(|| { + anyhow::anyhow!("--planner {planner} requires a positive --data-ingestion-interval-ms") + }) +} + +fn run_milp(args: &Args) -> anyhow::Result<()> { + let scrape_interval_ms = check_optimizer_flags(args, "milp")?; let config_path = args .input_config .as_deref() .ok_or_else(|| anyhow::anyhow!("--planner milp requires --input_config"))?; - let scrape_interval_ms = args - .data_ingestion_interval_ms - .ok_or_else(|| anyhow::anyhow!("--planner milp requires --data-ingestion-interval-ms"))?; - // Per-series sample rates divide by it. - anyhow::ensure!( - scrape_interval_ms > 0, - "--data-ingestion-interval-ms must be positive" - ); let config: ControllerConfig = serde_yaml::from_str(&std::fs::read_to_string(config_path)?)?; // Fail before solving: the plan is always written. reject_unwritable_queries(&config)?; @@ -315,36 +321,11 @@ fn run_milp(args: &Args) -> anyhow::Result<()> { } fn run_saved(args: &Args) -> anyhow::Result<()> { + let scrape_interval_ms = check_optimizer_flags(args, "saved")?; anyhow::ensure!( - args.input_config.is_none() && args.query_log.is_none(), - "--planner saved reads its queries from --workload-table, not --input_config or --query-log" - ); - anyhow::ensure!( - matches!(args.query_language, QueryLanguage::promql) - && args.prometheus_url.is_none() - && !args.enable_punting - && args.range_duration_ms == 0 - && args.step_ms == 0 - && args.clickhouse_url.is_none() - && args.clickhouse_database.is_none(), - "--planner saved takes only PromQL and none of --prometheus-url, --enable-punting, \ - --range-duration-ms, --step-ms, --clickhouse-url, --clickhouse-database" - ); - anyhow::ensure!( - args.workload_facts.is_none() - && args.atomic_costs.is_none() - && args.saturation_dir.is_none() - && args.w_cpu.is_none() - && args.w_mem.is_none(), - "--workload-facts, --atomic-costs, --saturation-dir, --w-cpu and --w-mem require \ - --planner milp" + args.input_config.is_none(), + "--planner saved reads its queries from --workload-table, not --input_config" ); - let scrape_interval_ms = args - .data_ingestion_interval_ms - .filter(|&ms| ms > 0) - .ok_or_else(|| { - anyhow::anyhow!("--planner saved requires a positive --data-ingestion-interval-ms") - })?; saved_plan_output( args.workload_table .as_deref() diff --git a/asap-planner-rs/src/optimizer/saved_plan.rs b/asap-planner-rs/src/optimizer/saved_plan.rs index 02ceecab..c383e6cd 100644 --- a/asap-planner-rs/src/optimizer/saved_plan.rs +++ b/asap-planner-rs/src/optimizer/saved_plan.rs @@ -8,7 +8,7 @@ use std::io::Read; use std::path::Path; use anyhow::{bail, ensure, Context}; -use rqe_optimizer::{AtomicCostEntry, Deployment}; +use rqe_optimizer::{AtomicCostEntry, Deployment, LabelSet}; use serde::Deserialize; use serde_json::{json, Value}; @@ -17,7 +17,7 @@ use crate::planner_output::PlannerOutput; use super::aqe_extractor::decompose_to_leaves; use super::milp::workload_from_config; -use super::milp_output::deployments_to_planner_output; +use super::milp_output::{deployments_to_planner_output, reject_unwritable_queries}; #[derive(Deserialize)] struct Table { @@ -76,8 +76,9 @@ struct Chosen { slide_secs: u64, /// Index of the plan's deployment. deployment: usize, - /// Present only when the deployment's grouping differs from the RQE's. - deployment_grouping: Option, + /// Present only when the deployment's grouping differs from the RQE's + /// (a roll-up), which the converter refuses. + deployment_grouping: Option, } /// Which plan of the file to write. @@ -164,7 +165,17 @@ pub fn saved_plan_output( selector.sla_ms, ); }; - let chosen: HashMap<&str, &Chosen> = plan.chosen.iter().map(|c| (c.rqe.as_str(), c)).collect(); + let mut chosen: HashMap<&str, &Chosen> = HashMap::new(); + for c in &plan.chosen { + ensure!( + chosen.insert(&c.rqe, c).is_none(), + "the plan picks RQE {:?} twice", + c.rqe + ); + } + let config = e2e_config(e2e)?; + // Before the leaf split, which turns an avg into two leaves. + reject_unwritable_queries(&config)?; // Leaf query string and repeat interval -> the plan RQE it reads. let mut rqe_of: HashMap<(String, u64), &str> = HashMap::new(); @@ -190,7 +201,6 @@ pub fn saved_plan_output( } } - let config = e2e_config(e2e)?; let workload = workload_from_config(&config, scrape_interval_ms)?; // Plan deployment index -> its position in `deployments`, in order of @@ -210,18 +220,32 @@ pub fn saved_plan_output( let pick = chosen .get(rqe) .with_context(|| format!("the plan doesn't serve RQE {rqe:?}"))?; + let (window_ms, slide_ms) = (pick.window_secs * 1000, pick.slide_secs * 1000); + // sketch-bench's `is_eligible` window rules: a pick that breaks + // them belongs to another workload table. ensure!( - pick.deployment_grouping.is_none(), - "RQE {rqe:?} is served by a roll-up, which the engine can't read yet" + slide_ms != 0 + && window_ms.is_multiple_of(slide_ms) + && raqe.lookback_ms.is_multiple_of(window_ms) + && raqe.interval_ms.is_multiple_of(slide_ms), + "RQE {rqe:?}'s {}s window sliding every {}s can't serve {leaf:?} \ + ({} ms lookback every {} ms); does --plan match --workload-table?", + pick.window_secs, + pick.slide_secs, + raqe.lookback_ms, + raqe.interval_ms ); let deployment = Deployment { capability: raqe.capability, metric: raqe.metric.clone(), spatial_filter: raqe.spatial_filter.clone(), - grouping_labels: raqe.grouping_labels.clone(), + grouping_labels: pick + .deployment_grouping + .clone() + .unwrap_or_else(|| raqe.grouping_labels.clone()), config: cost_entry(pick)?, - window_ms: pick.window_secs * 1000, - slide_ms: pick.slide_secs * 1000, + window_ms, + slide_ms, key_tracker: None, }; let at = *position.entry(pick.deployment).or_insert_with(|| { @@ -338,9 +362,13 @@ mod tests { } fn run(plan: Value, sla_ms: Option) -> anyhow::Result { + run_on(TABLE, plan, sla_ms) + } + + fn run_on(table: &str, plan: Value, sla_ms: Option) -> anyhow::Result { let dir = tempfile::tempdir().unwrap(); let (table_path, plan_path) = (dir.path().join("table.json"), dir.path().join("plan.json")); - std::fs::write(&table_path, TABLE).unwrap(); + std::fs::write(&table_path, table).unwrap(); std::fs::write(&plan_path, plan.to_string()).unwrap(); let selector = PlanSelector { method: "asap", @@ -384,12 +412,45 @@ mod tests { #[test] fn refuses_a_roll_up() { - let mut p50 = shared_p50(); - p50["deployment_grouping"] = json!(["job"]); - let err = refusal(plans(p50), None); + let mut plan = plans(shared_p50()); + plan["plans"][0]["chosen"][0]["deployment_grouping"] = json!(["job", "label_0"]); + let err = refusal(plan, None); assert!(err.contains("roll-up"), "{err}"); } + #[test] + fn refuses_a_pick_from_another_table() { + // A 7s window doesn't divide the 5m lookback. + let p50 = pick("p50", 0, "dd", json!({"alpha": 0.02}), 7); + let err = refusal(plans(p50), None); + assert!(err.contains("does --plan match --workload-table"), "{err}"); + } + + #[test] + fn refuses_an_avg_before_splitting_it() { + let table = TABLE.replace( + r#""sum by (job) (data)", "interval_secs": 1"#, + r#""avg by (job) (data)", "interval_secs": 1"#, + ); + let err = run_on(&table, plans(shared_p50()), None) + .err() + .expect("refused") + .to_string(); + assert!(err.contains("avg"), "{err}"); + assert!(!err.contains("leaves"), "{err}"); + } + + #[test] + fn refuses_an_rqe_picked_twice() { + let mut plan = plans(shared_p50()); + plan["plans"][0]["chosen"] + .as_array_mut() + .unwrap() + .push(shared_p50()); + let err = refusal(plan, None); + assert!(err.contains(r#"picks RQE "p50" twice"#), "{err}"); + } + #[test] fn refuses_one_deployment_with_two_sketches() { let p50 = pick("p50", 0, "dd", json!({"alpha": 0.05}), 300); diff --git a/asap-tools/experiments/experiment_run_e2e.py b/asap-tools/experiments/experiment_run_e2e.py index e7188f5b..9e653f8d 100644 --- a/asap-tools/experiments/experiment_run_e2e.py +++ b/asap-tools/experiments/experiment_run_e2e.py @@ -1,6 +1,5 @@ import os import json -import shutil import time import hydra @@ -322,10 +321,8 @@ def main(cfg: DictConfig): f"Skipping the planner: using configs from " f"{args.controller_prebuilt_output_dir}" ) - shutil.copytree( - args.controller_prebuilt_output_dir, - CONTROLLER_LOCAL_OUTPUT_DIR, - dirs_exist_ok=True, + config.copy_prebuilt_controller_output( + args.controller_prebuilt_output_dir, CONTROLLER_LOCAL_OUTPUT_DIR ) sync.rsync_streaming_configs( provider, diff --git a/asap-tools/experiments/experiment_utils/config.py b/asap-tools/experiments/experiment_utils/config.py index a1a363aa..b988cf98 100644 --- a/asap-tools/experiments/experiment_utils/config.py +++ b/asap-tools/experiments/experiment_utils/config.py @@ -5,6 +5,7 @@ import math import os +import shutil import copy import yaml from typing import Any, Dict, List, Tuple @@ -536,6 +537,29 @@ def get_prometheus_data_ingestion_interval_ms(prometheus_config): raise ValueError(f"Invalid scrape interval string: {s}") +PLANNER_OUTPUT_FILES = ("streaming_config.yaml", "inference_config.yaml") + + +def check_prebuilt_controller_output(src): + for name in PLANNER_OUTPUT_FILES: + path = os.path.join(src, name) + if not os.path.isfile(path): + raise ValueError(f"controller.prebuilt_output_dir has no {name}: {path}") + + +def copy_prebuilt_controller_output(src, dest): + """Replaces `dest` with the planner configs in `src`.""" + check_prebuilt_controller_output(src) + if os.path.realpath(src) == os.path.realpath(dest): + raise ValueError( + f"controller.prebuilt_output_dir is the experiment's own output dir: {dest}" + ) + shutil.rmtree(dest, ignore_errors=True) + os.makedirs(dest) + for name in PLANNER_OUTPUT_FILES: + shutil.copy2(os.path.join(src, name), dest) + + class Args: """Helper class to convert Hydra config to argparse-like namespace for backward compatibility.""" @@ -611,12 +635,7 @@ def __init__(self, cfg: DictConfig): self.controller_prebuilt_output_dir = hydra.utils.to_absolute_path( self.controller_prebuilt_output_dir ) - for name in ("streaming_config.yaml", "inference_config.yaml"): - path = os.path.join(self.controller_prebuilt_output_dir, name) - if not os.path.isfile(path): - raise ValueError( - f"controller.prebuilt_output_dir has no {name}: {path}" - ) + check_prebuilt_controller_output(self.controller_prebuilt_output_dir) # Aggregate cleanup configuration # Valid policies: "circular_buffer", "read_based", "no_cleanup" diff --git a/asap-tools/experiments/tests/test_prebuilt_controller_output.py b/asap-tools/experiments/tests/test_prebuilt_controller_output.py new file mode 100644 index 00000000..4dd227fc --- /dev/null +++ b/asap-tools/experiments/tests/test_prebuilt_controller_output.py @@ -0,0 +1,45 @@ +"""Tests for running e2e experiments on prebuilt planner configs.""" + +import os +import tempfile +import unittest + +from experiment_utils.config import copy_prebuilt_controller_output + + +class CopyPrebuiltControllerOutputTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.src = os.path.join(self.tmp.name, "src") + self.dest = os.path.join(self.tmp.name, "dest") + os.makedirs(self.src) + for name in ("streaming_config.yaml", "inference_config.yaml"): + with open(os.path.join(self.src, name), "w") as f: + f.write(name) + + def tearDown(self): + self.tmp.cleanup() + + def test_replaces_stale_output(self): + os.makedirs(self.dest) + with open(os.path.join(self.dest, "controller.log"), "w") as f: + f.write("stale") + copy_prebuilt_controller_output(self.src, self.dest) + self.assertEqual( + sorted(os.listdir(self.dest)), + ["inference_config.yaml", "streaming_config.yaml"], + ) + + def test_refuses_missing_config(self): + os.remove(os.path.join(self.src, "inference_config.yaml")) + with self.assertRaisesRegex(ValueError, "inference_config.yaml"): + copy_prebuilt_controller_output(self.src, self.dest) + + def test_refuses_own_output_dir(self): + with self.assertRaisesRegex(ValueError, "own output dir"): + copy_prebuilt_controller_output(self.src, self.src) + self.assertTrue(os.path.isfile(os.path.join(self.src, "streaming_config.yaml"))) + + +if __name__ == "__main__": + unittest.main() From 093e4a67eb754fb4b3878418815538b036d13828 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 13 Oct 2026 01:27:19 +0000 Subject: [PATCH 3/9] feat(asap-tools): run sketch-bench workload tables end to end with the paced feeder - exporters.exporter_list.paced_feeder: experiment_run_e2e.py runs one paced_feeder process per (sink, metric) on the worker nodes, pushing over remote write to the query engine's ingest (sketchdb mode) and/or Prometheus (other modes, or query_prometheus_too). It starts once its sinks accept writes, shares one --start-ms and seed, and fails the run if a feeder dies. Prometheus does not forward fed metrics to the query engine. - Prometheus (bare-metal and containerized) starts with --web.enable-remote-write-receiver when the feeder is configured. - client_options.align_to_interval: repetition k evaluates at floor((start - query_time_offset) / T) * T + k * T and is sent query_time_offset after that time. - sketch_bench_e2e/sketch_bench_e2e_configs.py writes an experiment_type config from a table's e2e block: one query group per interval, the feeder block, metrics, and controller.prebuilt_output_dir; structured queries only with a SQL file. - setup_dependencies.sh builds the feeder. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AnJ7LocWxguh3q4QWkCYae --- .github/workflows/python.yml | 3 + .../CONFIG_PARAMETERS_REFERENCE.md | 43 ++ asap-tools/experiments/constants.py | 2 + asap-tools/experiments/experiment_run_e2e.py | 47 +++ .../experiments/experiment_utils/core.py | 31 +- .../experiment_utils/services/__init__.py | 22 +- .../services/docker_prometheus.py | 16 +- .../services/fake_exporters.py | 10 +- .../experiment_utils/services/paced_feeder.py | 223 +++++++++++ .../experiment_utils/services/prometheus.py | 19 +- .../sketch_bench_e2e_configs.py | 239 ++++++++++++ .../experiments/tests/test_paced_feeder.py | 367 ++++++++++++++++++ .../tests/test_sketch_bench_e2e_configs.py | 204 ++++++++++ .../installation/common/setup_dependencies.sh | 1 + .../prometheus-client/classes/query_time.py | 40 ++ .../main_prometheus_client.py | 74 ++-- .../tests/test_query_time.py | 51 +++ 17 files changed, 1356 insertions(+), 36 deletions(-) create mode 100644 asap-tools/experiments/experiment_utils/services/paced_feeder.py create mode 100644 asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py create mode 100644 asap-tools/experiments/tests/test_paced_feeder.py create mode 100644 asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py create mode 100644 asap-tools/queriers/prometheus-client/classes/query_time.py create mode 100644 asap-tools/queriers/prometheus-client/tests/test_query_time.py diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 4a00b872..6b6cd229 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -92,6 +92,9 @@ jobs: - name: Type check with mypy working-directory: asap-tools/queriers/prometheus-client run: mypy . + - name: Run tests + working-directory: asap-tools/queriers/prometheus-client + run: python -m unittest discover -s tests -v test-utilities: needs: detect-changes diff --git a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md index 35a8afed..a85db47e 100644 --- a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md +++ b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md @@ -457,6 +457,44 @@ These parameters come from the `experiment_type` config group and are prefixed w - **Example**: `"gauge"` - **Note**: Counter metrics have `_total` suffix, gauge metrics don't +#### Paced Feeder Parameters + +`paced_feeder` runs `asap-tools/data-sources/paced-feeder` (built by +`installation/common/setup_dependencies.sh`) bare-metal on the worker nodes, +one process per (sink, metric). It pushes over remote write instead of being +scraped: in `sketchdb` mode to the query engine's ingest +(`streaming.remote_write`, which must have `parallelism: 1`), plus Prometheus +when the mode sets `query_prometheus_too`; in every other mode to Prometheus, +which then starts with `--web.enable-remote-write-receiver`. All processes +share the seed and table, so every system gets the same data. Its metrics are +listed under `metrics` with `exporter: paced_feeder`; Prometheus does not +forward them to the query engine. Generate a whole config from a sketch-bench +table with `sketch_bench_e2e/sketch_bench_e2e_configs.py`. + +#### `experiment_params.exporters.exporter_list.paced_feeder.workload_table` (string, required) +- **Description**: Local path of the sketch-bench workload table; copied to each feeder node +- **Example**: `/path/to/synthetic-templatesclassic-shared1.json` + +#### `experiment_params.exporters.exporter_list.paced_feeder.metrics` (list, required) +- **Description**: Metrics of the table's `workloads[0].e2e.metrics` to send; gauge or counter only +- **Example**: `[data, data_total]` + +#### `experiment_params.exporters.exporter_list.paced_feeder.seed` (int, required) +- **Description**: `--seed` of every feeder process +- **Example**: `42` + +#### `experiment_params.exporters.exporter_list.paced_feeder.duration_secs` (int, required) +- **Description**: Seconds of data to send. Must be at least `flow.steady_state_wait` plus the query groups' running time +- **Example**: `4000` + +#### `experiment_params.exporters.exporter_list.paced_feeder.samples_per_sec` (int, optional) +- **Description**: Overrides the table's `samples_per_sec_per_series` +- **Example**: `20` + +#### `experiment_params.exporters.exporter_list.paced_feeder.extra_flags` (string, optional) +- **Description**: Appended to every feeder command line +- **Example**: `"--concurrency 32"` + ### Query Group Configuration #### `experiment_params.query_groups` (list, required) @@ -500,6 +538,11 @@ These parameters come from the `experiment_type` config group and are prefixed w - **Example**: `15` - **Usage**: Ensures data is available for lookback queries +#### `experiment_params.query_groups[].client_options.align_to_interval` (boolean, optional) +- **Description**: Evaluate repetition k at `t_k = floor((start - query_time_offset) / T) * T + k * T`, with `T = repetition_delay_ms` (whole seconds), sending it at wall-clock `t_k + query_time_offset` +- **Default**: `false` +- **Usage**: Every evaluation time is a multiple of `T`, so each system answers the same windows; `query_time_offset` is how long the data of the window ending at `t_k` gets to arrive. Overrides the query engine's per-aggregation alignment in sketchdb mode + #### `experiment_params.query_groups[].client_options.starting_delay` (int, required) - **Description**: Initial delay in seconds before starting query execution - **Example**: `610` diff --git a/asap-tools/experiments/constants.py b/asap-tools/experiments/constants.py index 487e7e2c..2faf6967 100644 --- a/asap-tools/experiments/constants.py +++ b/asap-tools/experiments/constants.py @@ -42,6 +42,8 @@ PROMETHEUS_CONFIG_DIR = "prometheus_config" PROMETHEUS_CONFIG_FILE = "prometheus.yml" PROMETHEUS_REMOTE_WRITE_ENDPOINT = "/api/v1/write" +# Lets Prometheus accept samples pushed to PROMETHEUS_REMOTE_WRITE_ENDPOINT. +PROMETHEUS_REMOTE_WRITE_RECEIVER_FLAG = "--web.enable-remote-write-receiver" HTTP_READINESS_CURL_TIMEOUT_SECONDS = 5 QUERY_ENGINE_RUNTIME_INFO_ENDPOINT = "/api/v1/status/runtimeinfo" diff --git a/asap-tools/experiments/experiment_run_e2e.py b/asap-tools/experiments/experiment_run_e2e.py index 9e653f8d..f1a6ef1a 100644 --- a/asap-tools/experiments/experiment_run_e2e.py +++ b/asap-tools/experiments/experiment_run_e2e.py @@ -25,6 +25,10 @@ DockerPrometheusService, DockerVictoriaMetricsService, SystemExportersService, + PACED_FEEDER_EXPORTER, + PacedFeederService, + check_paced_feeder_config, + paced_feeder_sinks, ) from experiment_utils.services.misc import DiscoveryBackend @@ -188,6 +192,22 @@ def main(cfg: DictConfig): cfg.experiment_params ) + paced_feeder_service = None + paced_feeder_config = exporter_config["exporter_list"].get(PACED_FEEDER_EXPORTER) + if paced_feeder_config is not None: + paced_feeder_config = OmegaConf.to_container(paced_feeder_config, resolve=True) + check_paced_feeder_config( + paced_feeder_config, + parallelism=args.parallelism, + min_feed_secs=(0 if skip_querying else args.steady_state_wait) + + minimum_experiment_running_time, + ) + paced_feeder_service = PacedFeederService(provider, args) + modes_by_name = { + m["mode"]: m + for m in OmegaConf.to_container(cfg.experiment_params.experiment, resolve=True) + } + for experiment_mode in experiment_modes: print(f"Running experiment mode: {experiment_mode}") experiment_output_dir = os.path.join( @@ -228,6 +248,8 @@ def main(cfg: DictConfig): exporter_service.stop() deathstar_service.stop() prometheus_service.reset() + if paced_feeder_service is not None: + paced_feeder_service.stop() # Also stop avalanche exporters if they were started if config.check_exporter_and_queries_exist("avalanche", cfg.experiment_params): @@ -469,6 +491,28 @@ def main(cfg: DictConfig): ), f"Expected PrometheusService but got {type(prometheus_service).__name__}" prometheus_service.start(experiment_output_dir) + # Pushes to its sinks, so it starts once they accept writes. + if paced_feeder_service is not None: + prometheus_url = ( + f"http://{args.remote_write_ip}:" + f"{prometheus_service.get_query_endpoint_port()}" + f"{constants.PROMETHEUS_REMOTE_WRITE_ENDPOINT}" + ) + sinks = paced_feeder_sinks( + modes_by_name[experiment_mode], + query_engine_url=f"http://{args.remote_write_ip}:" + f"{args.remote_write_base_port}{args.remote_write_path}", + prometheus_url=prometheus_url, + ) + if prometheus_url in sinks.values(): + prometheus_service.wait_until_ready() + paced_feeder_service.start( + config=paced_feeder_config, + sinks=sinks, + experiment_output_dir=experiment_output_dir, + local_experiment_dir=local_experiment_dir, + ) + # Start Prometheus throughput monitoring if enabled if args.throughput_prometheus: prometheus_throughput_monitor = PrometheusThroughputMonitor( @@ -546,6 +590,9 @@ def main(cfg: DictConfig): # Skip teardown if the no_teardown flag is set if not args.no_teardown: + # Before its sinks go away, so its log shows no spurious errors. + if paced_feeder_service is not None: + paced_feeder_service.stop() if experiment_mode == constants.SKETCHDB_EXPERIMENT_NAME: query_engine_service.stop() diff --git a/asap-tools/experiments/experiment_utils/core.py b/asap-tools/experiments/experiment_utils/core.py index 2bb24960..2a840130 100644 --- a/asap-tools/experiments/experiment_utils/core.py +++ b/asap-tools/experiments/experiment_utils/core.py @@ -134,6 +134,17 @@ def read_exporter_config(experiment_params: DictConfig) -> Tuple[Optional[Dict], "cluster_data_exporter with parts_mode='part-index' requires 'part_index'", ) + from experiment_utils.services.paced_feeder import ( + PACED_FEEDER_EXPORTER, + PACED_FEEDER_REQUIRED_KEYS, + ) + + if PACED_FEEDER_EXPORTER in exporters_config.exporter_list: + feeder_config = exporters_config.exporter_list[PACED_FEEDER_EXPORTER] + missing_keys = [k for k in PACED_FEEDER_REQUIRED_KEYS if k not in feeder_config] + if missing_keys: + return None, f"Missing keys in paced_feeder section: {missing_keys}" + return exporters_config, "" @@ -186,9 +197,25 @@ def call_generate_prometheus_config( import generate_prometheus_config import generate_victoriametrics_config + from experiment_utils.services.paced_feeder import paced_feeder_metric_names + + # The paced feeder writes its metrics to the query engine itself, so + # Prometheus forwarding them too would send them twice. + fed_metrics = paced_feeder_metric_names(cfg.experiment_params) + metrics_to_remote_write = [ + m + for m in get_metrics_to_remote_write(cfg.experiment_params) + if m not in fed_metrics + ] + # With only fed metrics there is nothing for Prometheus to forward. (An + # empty list without the feeder keeps meaning "forward everything".) + forward_to_query_engine = bool(metrics_to_remote_write) or not fed_metrics # Set up remote write if this is a SketchDB experiment - if experiment_mode == sketchdb_experiment_name and sketchdb_experiment_name: - metrics_to_remote_write = get_metrics_to_remote_write(cfg.experiment_params) + if ( + experiment_mode == sketchdb_experiment_name + and sketchdb_experiment_name + and forward_to_query_engine + ): args.remote_write_metric_names = metrics_to_remote_write # Build remote_write_url from experiment params diff --git a/asap-tools/experiments/experiment_utils/services/__init__.py b/asap-tools/experiments/experiment_utils/services/__init__.py index d560b97d..e4581dbb 100644 --- a/asap-tools/experiments/experiment_utils/services/__init__.py +++ b/asap-tools/experiments/experiment_utils/services/__init__.py @@ -34,6 +34,12 @@ ControllerService, ) from .grafana import GrafanaService +from .paced_feeder import ( + PACED_FEEDER_EXPORTER, + PacedFeederService, + check_paced_feeder_config, + paced_feeder_sinks, +) from .clickhouse_service import ClickHouseService, ClickHouseDataLoaderService @@ -71,6 +77,10 @@ def create_prometheus_service(cfg, provider, num_nodes: int, node_offset: int): ) monitoring = cfg.experiment_params.monitoring + # The paced feeder pushes over remote write instead of being scraped. + remote_write_receiver = ( + PACED_FEEDER_EXPORTER in cfg.experiment_params.exporters.exporter_list + ) # Validate required fields if not hasattr(monitoring, "tool"): @@ -108,7 +118,9 @@ def create_prometheus_service(cfg, provider, num_nodes: int, node_offset: int): # Create appropriate service based on deployment mode if monitoring.deployment_mode == "containerized": if monitoring.tool == "prometheus": - return DockerPrometheusService(provider, num_nodes, node_offset) + return DockerPrometheusService( + provider, num_nodes, node_offset, remote_write_receiver + ) elif monitoring.tool == "victoriametrics": return DockerVictoriaMetricsService(provider, num_nodes, node_offset) else: # bare_metal @@ -117,7 +129,9 @@ def create_prometheus_service(cfg, provider, num_nodes: int, node_offset: int): "ERROR: VictoriaMetrics only supports containerized deployment. " "Use tool: prometheus for bare_metal deployment." ) - return PrometheusService(provider, num_nodes, node_offset) + return PrometheusService( + provider, num_nodes, node_offset, remote_write_receiver + ) __all__ = [ @@ -147,4 +161,8 @@ def create_prometheus_service(cfg, provider, num_nodes: int, node_offset: int): "create_prometheus_service", "ClickHouseService", "ClickHouseDataLoaderService", + "PACED_FEEDER_EXPORTER", + "PacedFeederService", + "check_paced_feeder_config", + "paced_feeder_sinks", ] diff --git a/asap-tools/experiments/experiment_utils/services/docker_prometheus.py b/asap-tools/experiments/experiment_utils/services/docker_prometheus.py index 88c0e519..8475344c 100644 --- a/asap-tools/experiments/experiment_utils/services/docker_prometheus.py +++ b/asap-tools/experiments/experiment_utils/services/docker_prometheus.py @@ -6,14 +6,22 @@ from .base import DockerServiceBase from experiment_utils.providers.base import InfrastructureProvider -from constants import PROMETHEUS_CONFIG_DIR, PROMETHEUS_CONFIG_FILE +from constants import ( + PROMETHEUS_CONFIG_DIR, + PROMETHEUS_CONFIG_FILE, + PROMETHEUS_REMOTE_WRITE_RECEIVER_FLAG, +) class DockerPrometheusService(DockerServiceBase): """Docker-based Prometheus service with resource constraints.""" def __init__( - self, provider: InfrastructureProvider, num_nodes: int, node_offset: int + self, + provider: InfrastructureProvider, + num_nodes: int, + node_offset: int, + remote_write_receiver: bool = False, ): """ Initialize Docker Prometheus service. @@ -22,9 +30,11 @@ def __init__( provider: Infrastructure provider for node communication and management num_nodes: Number of nodes to manage node_offset: Starting node index offset + remote_write_receiver: Accept samples on /api/v1/write """ super().__init__(provider, num_nodes, node_offset) self.container_name = "prometheus-container" + self.remote_write_receiver = remote_write_receiver def get_container_name(self) -> str: """Get the Docker container name.""" @@ -100,6 +110,8 @@ def start( "--web.enable-lifecycle", ] ) + if self.remote_write_receiver: + docker_cmd_parts.append(PROMETHEUS_REMOTE_WRITE_RECEIVER_FLAG) docker_cmd = " ".join(docker_cmd_parts) diff --git a/asap-tools/experiments/experiment_utils/services/fake_exporters.py b/asap-tools/experiments/experiment_utils/services/fake_exporters.py index ade2eaef..e080084f 100644 --- a/asap-tools/experiments/experiment_utils/services/fake_exporters.py +++ b/asap-tools/experiments/experiment_utils/services/fake_exporters.py @@ -42,7 +42,7 @@ def get_fake_exporter_seed( return derived_seed -def get_fake_exporter_target_nodes(args) -> List[int]: +def get_exporter_target_nodes(args) -> List[int]: """Return exporter nodes, preserving the single-node local fallback.""" worker_nodes = args.get_node_range(include_coordinator=False) return worker_nodes or [args.get_coordinator_node()] @@ -429,7 +429,7 @@ def _start_bare_metal( dataset = config["dataset"] commands_by_port: List[List[Tuple[int, str]]] = [] - target_nodes = get_fake_exporter_target_nodes(self.args) + target_nodes = get_exporter_target_nodes(self.args) for port_ordinal in range(num_ports): port_commands = [] for worker_ordinal, node_idx in enumerate(target_nodes): @@ -488,7 +488,7 @@ def _start_containerized( # Build docker run commands for each port docker_run_cmds: List[Tuple[int, str]] = [] container_names: List[str] = [] - target_nodes = get_fake_exporter_target_nodes(self.args) + target_nodes = get_exporter_target_nodes(self.args) for worker_ordinal, node_idx in enumerate(target_nodes): for port_ordinal in range(num_ports): @@ -579,7 +579,7 @@ def _stop_bare_metal(self, **kwargs) -> None: **kwargs: Additional configuration (currently unused) """ cmd = "pkill -f fake_exporter" - target_nodes = get_fake_exporter_target_nodes(self.args) + target_nodes = get_exporter_target_nodes(self.args) self.provider.execute_command_parallel( node_idxs=target_nodes, cmd=cmd, @@ -591,7 +591,7 @@ def _stop_bare_metal(self, **kwargs) -> None: def _stop_containerized(self, **kwargs) -> None: """Stop fake exporters using containerized deployment.""" - target_nodes = get_fake_exporter_target_nodes(self.args) + target_nodes = get_exporter_target_nodes(self.args) try: if self.container_names is not None and len(self.container_names) > 0: # Stop and remove containers by name diff --git a/asap-tools/experiments/experiment_utils/services/paced_feeder.py b/asap-tools/experiments/experiment_utils/services/paced_feeder.py new file mode 100644 index 00000000..c01abb3e --- /dev/null +++ b/asap-tools/experiments/experiment_utils/services/paced_feeder.py @@ -0,0 +1,223 @@ +""" +Paced feeder service: pushes a sketch-bench workload table's metrics in real +time over Prometheus remote write, one feeder process per (sink, metric). + +Unlike the fake exporters, nothing is scraped: the feeder writes straight to +ASAPQuery's ingest and/or Prometheus' remote-write receiver. Every process gets +the same seed and table, so each system receives identical data. +""" + +import json +import math +import os +import time +from typing import Any, Dict, List, Tuple + +import constants +from .base import BaseService +from .fake_exporters import get_exporter_target_nodes +from experiment_utils import sync +from experiment_utils.providers.base import InfrastructureProvider + +PACED_FEEDER_EXPORTER = "paced_feeder" +PACED_FEEDER_REQUIRED_KEYS = ("workload_table", "metrics", "seed", "duration_secs") +# Metric types a remote-write sink (ASAPQuery ingest, Prometheus) accepts. +REMOTE_WRITE_METRIC_TYPES = ("gauge", "counter") + +PACED_FEEDER_BINARY = "./target/release/paced_feeder" +# The brackets keep pgrep/pkill from matching the shell that runs them. +PACED_FEEDER_PROCESS_PATTERN = "[t]arget/release/paced_feeder" +# Feeders launched one after another over SSH all start at this much later +# event time, so the sinks of one mode get the same timestamps. +PACED_FEEDER_START_LEAD_SECS = 5 +# A feeder stops after its first tick if every request fails (e.g. a wrong +# URL), so check the processes one tick after the shared start. +PACED_FEEDER_STARTUP_CHECK_SECS = PACED_FEEDER_START_LEAD_SECS + 2 + +SINK_QUERY_ENGINE = "query_engine" +SINK_PROMETHEUS = "prometheus" + + +def paced_feeder_metric_names(experiment_params) -> List[str]: + """Metrics the paced feeder sends, per the experiment's `metrics` list.""" + return [ + m["metric"] + for m in experiment_params.get("metrics", []) + if m["exporter"] == PACED_FEEDER_EXPORTER + ] + + +def check_paced_feeder_config( + config: Dict[str, Any], parallelism: int, min_feed_secs: float +) -> None: + """Fail before the experiment starts on a config the feeder would reject + or one that stops sending before the queries end.""" + with open(config["workload_table"]) as f: + (workload,) = json.load(f)["workloads"] + table_metrics = workload["e2e"]["metrics"] + for metric in config["metrics"]: + if metric not in table_metrics: + raise ValueError( + f"paced_feeder metric {metric!r} is not in " + f"{config['workload_table']} (has {sorted(table_metrics)})" + ) + metric_type = table_metrics[metric]["type"] + if metric_type not in REMOTE_WRITE_METRIC_TYPES: + raise ValueError( + f"paced_feeder metric {metric!r} has type {metric_type!r}; " + f"remote write takes only {REMOTE_WRITE_METRIC_TYPES}" + ) + if parallelism != 1: + raise ValueError( + "paced_feeder sends to one ingest port; set streaming.parallelism=1" + ) + if config["duration_secs"] < min_feed_secs: + raise ValueError( + f"paced_feeder.duration_secs={config['duration_secs']} ends before the " + f"queries do: needs at least {min_feed_secs} " + "(flow.steady_state_wait + the query groups' running time)" + ) + + +def paced_feeder_sinks( + mode: Dict[str, Any], query_engine_url: str, prometheus_url: str +) -> Dict[str, str]: + """Sink name -> remote-write URL for one experiment mode. + + sketchdb feeds ASAPQuery's ingest, plus Prometheus when the mode also + queries it; every other mode queries Prometheus only. + """ + if mode["mode"] != constants.SKETCHDB_EXPERIMENT_NAME: + return {SINK_PROMETHEUS: prometheus_url} + sinks = {SINK_QUERY_ENGINE: query_engine_url} + if mode.get("query_prometheus_too", False): + sinks[SINK_PROMETHEUS] = prometheus_url + return sinks + + +def paced_feeder_command( + config: Dict[str, Any], + workload_table: str, + url: str, + metric: str, + start_ms: int, + stats_out: str, +) -> str: + cmd = ( + f"{PACED_FEEDER_BINARY} --sink remote-write --url {url}" + f" --workload-table {workload_table} --metric {metric}" + f" --seed {config['seed']} --duration-secs {config['duration_secs']}" + f" --start-ms {start_ms} --stats-out {stats_out}" + ) + if config.get("samples_per_sec") is not None: + cmd += f" --samples-per-sec {config['samples_per_sec']}" + if config.get("extra_flags"): + cmd += f" {config['extra_flags']}" + return cmd + + +class PacedFeederService(BaseService): + """Runs paced_feeder processes bare-metal, spread over the worker nodes + (the coordinator in local mode).""" + + def __init__(self, provider: InfrastructureProvider, args): + super().__init__(provider) + self.target_nodes = get_exporter_target_nodes(args) + self.cmd_dir = os.path.join( + provider.get_home_dir(), + "code", + "asap-tools", + "data-sources", + "paced-feeder", + ) + + def start( + self, + config: Dict[str, Any], + sinks: Dict[str, str], + experiment_output_dir: str, + local_experiment_dir: str, + **kwargs, + ) -> None: + output_dir = os.path.join(experiment_output_dir, "paced_feeder_output") + # One shared event-time origin for every process of this mode. + start_ms = (math.ceil(time.time()) + PACED_FEEDER_START_LEAD_SECS) * 1000 + + jobs = [(sink, metric) for sink in sinks for metric in config["metrics"]] + job_nodes = [ + self.target_nodes[i % len(self.target_nodes)] for i in range(len(jobs)) + ] + used_nodes = sorted(set(job_nodes)) + + for node_idx in used_nodes: + self.provider.execute_command( + node_idx=node_idx, + cmd=f"test -x {PACED_FEEDER_BINARY}", + cmd_dir=self.cmd_dir, + nohup=False, + popen=False, + ignore_errors=False, + ) + # Same path on every node; also creates output_dir. + workload_table = sync.rsync_dataset_file( + self.provider, config["workload_table"], output_dir, node_idx + ) + + node_cmds: List[Tuple[int, str]] = [] + for node_idx, (sink, metric) in zip(job_nodes, jobs): + name = f"{sink}_{metric}" + cmd = paced_feeder_command( + config, + workload_table, + sinks[sink], + metric, + start_ms, + os.path.join(output_dir, f"{name}_stats.json"), + ) + node_cmds.append( + (node_idx, f"{cmd} > {os.path.join(output_dir, name)}.out 2>&1 &") + ) + + local_config_dir = os.path.join(local_experiment_dir, "paced_feeder_config") + os.makedirs(local_config_dir, exist_ok=True) + with open(os.path.join(local_config_dir, "cmds.sh"), "w") as f: + f.write("\n".join(cmd for _, cmd in node_cmds) + "\n") + + for node_idx, cmd in node_cmds: + self.provider.execute_command( + node_idx=node_idx, + cmd=cmd, + cmd_dir=self.cmd_dir, + nohup=True, + popen=False, + ignore_errors=False, + ) + + time.sleep(PACED_FEEDER_STARTUP_CHECK_SECS) + for node_idx in used_nodes: + expected = job_nodes.count(node_idx) + # No $(...): the SSH command is double-quoted on this side. + result = self.provider.execute_command( + node_idx=node_idx, + cmd=f"pgrep -fc '{PACED_FEEDER_PROCESS_PATTERN}' | grep -qx {expected}", + cmd_dir=None, + nohup=False, + popen=False, + ignore_errors=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"paced_feeder: expected {expected} processes running on " + f"node {node_idx}; see the .out files in {output_dir}" + ) + + def stop(self, **kwargs) -> None: + self.provider.execute_command_parallel( + node_idxs=self.target_nodes, + cmd=f"pkill -f '{PACED_FEEDER_PROCESS_PATTERN}'", + cmd_dir=None, + nohup=False, + popen=True, + wait=True, + ignore_errors=True, + ) diff --git a/asap-tools/experiments/experiment_utils/services/prometheus.py b/asap-tools/experiments/experiment_utils/services/prometheus.py index 07d54137..57697a7e 100644 --- a/asap-tools/experiments/experiment_utils/services/prometheus.py +++ b/asap-tools/experiments/experiment_utils/services/prometheus.py @@ -8,14 +8,22 @@ from .base import BaseService from experiment_utils.providers.base import InfrastructureProvider -from constants import PROMETHEUS_CONFIG_DIR, PROMETHEUS_CONFIG_FILE +from constants import ( + PROMETHEUS_CONFIG_DIR, + PROMETHEUS_CONFIG_FILE, + PROMETHEUS_REMOTE_WRITE_RECEIVER_FLAG, +) class PrometheusService(BaseService): """Service for managing Prometheus operations.""" def __init__( - self, provider: InfrastructureProvider, num_nodes: int, node_offset: int + self, + provider: InfrastructureProvider, + num_nodes: int, + node_offset: int, + remote_write_receiver: bool = False, ): """ Initialize Prometheus service. @@ -24,10 +32,12 @@ def __init__( provider: Infrastructure provider for node communication and management num_nodes: Number of nodes to manage node_offset: Starting node index offset + remote_write_receiver: Accept samples on /api/v1/write """ super().__init__(provider) self.num_nodes = num_nodes self.node_offset = node_offset + self.remote_write_receiver = remote_write_receiver def get_query_endpoint_port(self) -> int: """Get the query endpoint port for Prometheus.""" @@ -89,7 +99,10 @@ def _start_prometheus(self, experiment_output_dir: str) -> None: time.sleep(3) # Start prometheus - cmd = f"./prometheus --config.file={PROMETHEUS_CONFIG_FILE} > /dev/null 2>&1 < /dev/null &" + flags = f"--config.file={PROMETHEUS_CONFIG_FILE}" + if self.remote_write_receiver: + flags += f" {PROMETHEUS_REMOTE_WRITE_RECEIVER_FLAG}" + cmd = f"./prometheus {flags} > /dev/null 2>&1 < /dev/null &" self.provider.execute_command( node_idx=self.node_offset, cmd=cmd, diff --git a/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py b/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py new file mode 100644 index 00000000..27a5eb9d --- /dev/null +++ b/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py @@ -0,0 +1,239 @@ +"""Write an experiment_run_e2e.py config for a sketch-bench workload table. + +The table's `workloads[0].e2e` block lists the end-to-end queries and the +metrics the paced feeder generates. The config has: + +- one query group per repeat interval T (`interval_secs`), repeating every T + with evaluation times aligned to multiples of T (`align_to_interval`). PromQL + entries keep the table's order, as in `asap-planner --planner saved`. +- structured entries (no `promql`, the multi-grouping templates) as SQL query + groups, only when `--sql-queries` gives their SQL; otherwise they are + skipped with a note. The file holds one `;`-terminated statement per + structured entry, in table order. +- the table's gauge and counter metrics, sent by the paced feeder (records + metrics are skipped with a note: remote write cannot carry them). +- `controller.prebuilt_output_dir`, when given, so ASAPQuery runs those + planner configs (e.g. from `asap-planner --planner saved`). + +Usage (from asap-tools/experiments): + python sketch_bench_e2e/sketch_bench_e2e_configs.py \\ + --workload-table /.../synthetic-templatesclassic-shared1.json \\ + --seed 42 --starting-delay-secs 120 --query-duration-secs 3600 \\ + [--prebuilt-output-dir DIR] [--sql-queries FILE] +Then: + python experiment_run_e2e.py experiment_type=sketch_bench_e2e/
... +""" + +import argparse +import json +import os +import sys +from typing import Dict, List, Optional, Tuple + +import yaml + +EXPERIMENTS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, EXPERIMENTS_DIR) + +from experiment_utils.config import ( # noqa: E402 + _load_sql_queries, + check_prebuilt_controller_output, +) +from experiment_utils.services.paced_feeder import ( # noqa: E402 + PACED_FEEDER_EXPORTER, + REMOTE_WRITE_METRIC_TYPES, +) + +DEFAULT_OUTPUT_DIR = os.path.join( + EXPERIMENTS_DIR, "config", "experiment_type", "sketch_bench_e2e" +) +SERVERS = [ + {"name": "prometheus", "url": "http://localhost:9090"}, + {"name": "sketchdb", "url": "http://localhost:8088"}, +] +MODES = [ + {"mode": "sketchdb", "server": "sketchdb"}, + {"mode": "baseline", "server": "prometheus"}, +] +# Seconds the window ending at an evaluation time gets for its data to arrive. +DEFAULT_QUERY_TIME_OFFSET_SECS = 10 +# Feeder time beyond the query phase: covers flow.steady_state_wait (60 s in +# config.yaml) and service startup. The runner checks the total. +FEEDER_SLACK_SECS = 300 + + +def load_e2e(table_path: str) -> dict: + with open(table_path) as f: + (workload,) = json.load(f)["workloads"] + return workload["e2e"] + + +def by_interval(entries: List[Tuple[int, str]]) -> Dict[int, List[str]]: + groups: Dict[int, List[str]] = {} + for interval_secs, query in entries: + groups.setdefault(interval_secs, []).append(query) + return dict(sorted(groups.items())) + + +def build_config( + table_path: str, + seed: int, + starting_delay_secs: int, + query_duration_secs: int, + query_time_offset_secs: int = DEFAULT_QUERY_TIME_OFFSET_SECS, + feeder_duration_secs: Optional[int] = None, + samples_per_sec: Optional[int] = None, + prebuilt_output_dir: Optional[str] = None, + sql_queries_file: Optional[str] = None, +) -> Tuple[dict, List[str]]: + """Return (config, notes on what was skipped).""" + e2e = load_e2e(table_path) + notes: List[str] = [] + + promql = [ + (q["interval_secs"], q["promql"]) for q in e2e["queries"] if "promql" in q + ] + structured = [q for q in e2e["queries"] if "promql" not in q] + sql: List[Tuple[int, str]] = [] + if structured and sql_queries_file is None: + notes.append( + f"skipped {len(structured)} structured (multi-grouping) queries: " + "no --sql-queries file" + ) + elif structured: + statements = _load_sql_queries(sql_queries_file) + if len(statements) != len(structured): + raise ValueError( + f"{sql_queries_file} has {len(statements)} statements; the table " + f"has {len(structured)} structured queries" + ) + sql = [(q["interval_secs"], s) for q, s in zip(structured, statements)] + + query_groups = [] + for queries_by_interval in (by_interval(promql), by_interval(sql)): + for interval_secs, queries in queries_by_interval.items(): + if query_duration_secs < interval_secs: + raise ValueError( + f"--query-duration-secs {query_duration_secs} is shorter than " + f"the {interval_secs} s repeat interval" + ) + query_groups.append( + { + "id": len(query_groups) + 1, + "queries": queries, + "repetition_delay_ms": interval_secs * 1000, + "client_options": { + "repetitions": query_duration_secs // interval_secs, + "starting_delay": starting_delay_secs, + "query_time_offset": query_time_offset_secs, + "align_to_interval": True, + }, + } + ) + if not query_groups: + raise ValueError(f"{table_path}: no queries to run") + + metrics = [] + for name, desc in e2e["metrics"].items(): + if desc["type"] not in REMOTE_WRITE_METRIC_TYPES: + notes.append(f"skipped {desc['type']} metric {name!r}: not remote-writable") + continue + metrics.append( + { + "metric": name, + "labels": [label["name"] for label in desc["labels"]], + "exporter": PACED_FEEDER_EXPORTER, + } + ) + + feeder = { + "workload_table": os.path.abspath(table_path), + "metrics": [m["metric"] for m in metrics], + "seed": seed, + "duration_secs": ( + feeder_duration_secs + if feeder_duration_secs is not None + else starting_delay_secs + query_duration_secs + FEEDER_SLACK_SECS + ), + } + if samples_per_sec is not None: + feeder["samples_per_sec"] = samples_per_sec + + config: dict = { + "experiment_params": { + "experiment": MODES, + "monitoring": {"tool": "prometheus", "deployment_mode": "bare_metal"}, + "servers": SERVERS, + "exporters": { + "only_start_if_queries_exist": False, + "exporter_list": {PACED_FEEDER_EXPORTER: feeder}, + }, + "query_groups": query_groups, + "metrics": metrics, + } + } + if prebuilt_output_dir is not None: + prebuilt_output_dir = os.path.abspath(prebuilt_output_dir) + check_prebuilt_controller_output(prebuilt_output_dir) + config["controller"] = {"prebuilt_output_dir": prebuilt_output_dir} + return config, notes + + +def write_config(config: dict, table_path: str, output_path: str) -> None: + os.makedirs(os.path.dirname(output_path), exist_ok=True) + with open(output_path, "w") as f: + f.write("# @package _global_\n") + f.write(f"# Generated by sketch_bench_e2e_configs.py from {table_path}\n") + yaml.safe_dump(config, f, sort_keys=False) + + +def main() -> None: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--workload-table", required=True) + parser.add_argument("--seed", type=int, required=True) + parser.add_argument("--starting-delay-secs", type=int, required=True) + parser.add_argument( + "--query-duration-secs", + type=int, + required=True, + help="query phase length; each group repeats this // T times", + ) + parser.add_argument( + "--query-time-offset-secs", + type=int, + default=DEFAULT_QUERY_TIME_OFFSET_SECS, + ) + parser.add_argument( + "--feeder-duration-secs", + type=int, + help=f"default: starting delay + query duration + {FEEDER_SLACK_SECS}", + ) + parser.add_argument("--samples-per-sec", type=int) + parser.add_argument("--prebuilt-output-dir") + parser.add_argument("--sql-queries") + parser.add_argument("--output", help="default: /
.yaml") + args = parser.parse_args() + + config, notes = build_config( + args.workload_table, + seed=args.seed, + starting_delay_secs=args.starting_delay_secs, + query_duration_secs=args.query_duration_secs, + query_time_offset_secs=args.query_time_offset_secs, + feeder_duration_secs=args.feeder_duration_secs, + samples_per_sec=args.samples_per_sec, + prebuilt_output_dir=args.prebuilt_output_dir, + sql_queries_file=args.sql_queries, + ) + stem = os.path.splitext(os.path.basename(args.workload_table))[0] + output = args.output or os.path.join(DEFAULT_OUTPUT_DIR, f"{stem}.yaml") + write_config(config, args.workload_table, output) + for note in notes: + print(f"note: {note}") + print(output) + + +if __name__ == "__main__": + main() diff --git a/asap-tools/experiments/tests/test_paced_feeder.py b/asap-tools/experiments/tests/test_paced_feeder.py new file mode 100644 index 00000000..6e55fdc9 --- /dev/null +++ b/asap-tools/experiments/tests/test_paced_feeder.py @@ -0,0 +1,367 @@ +"""Tests for running the paced feeder from the e2e runner.""" + +import json +import os +import subprocess +import tempfile +import unittest +from unittest import mock + +from omegaconf import OmegaConf + +import constants +from experiment_utils.core import call_generate_prometheus_config, read_exporter_config +from experiment_utils.services import create_prometheus_service +from experiment_utils.services.docker_prometheus import DockerPrometheusService +from experiment_utils.services.paced_feeder import ( + PacedFeederService, + check_paced_feeder_config, + paced_feeder_command, + paced_feeder_sinks, +) +from experiment_utils.services.prometheus import PrometheusService + +QE_URL = "http://10.10.1.1:8080/api/v1/write" +PROM_URL = "http://10.10.1.1:9090/api/v1/write" + +# workloads[0].e2e.metrics of synthetic-templatesclassic-shared1.json, plus a +# records metric from the multi-grouping tables. +LABELS = [ + {"cardinality": 10, "name": "job"}, + {"cardinality": 1000, "child_of": "job", "name": "label_0"}, +] +E2E_METRICS = { + "data": { + "labels": LABELS, + "samples_per_sec_per_series": 200, + "type": "gauge", + "values": {"dist": {"pareto": 2.0}, "value": {"keys": 10000, "zipf": 1.1}}, + }, + "data_total": { + "labels": LABELS, + "samples_per_sec_per_series": 200, + "type": "counter", + "values": {"increment": {"keys": 10000, "zipf": 1.1}}, + }, + "flows": { + "labels": [{"cardinality": 3, "name": "proto", "skew": 0.0}], + "records_per_sec": 4000000, + "type": "records", + "values": {"src_ip": {"keys": 1000000, "zipf": 0.0}}, + }, +} + + +def write_table(directory, queries=()): + path = os.path.join(directory, "table.json") + with open(path, "w") as f: + json.dump( + {"workloads": [{"e2e": {"metrics": E2E_METRICS, "queries": queries}}]}, f + ) + return path + + +def feeder_config(table, **overrides): + return { + "workload_table": table, + "metrics": ["data", "data_total"], + "seed": 42, + "duration_secs": 600, + **overrides, + } + + +class PacedFeederCommandTest(unittest.TestCase): + def test_command_line(self): + cmd = paced_feeder_command( + feeder_config("/t/table.json"), + "/out/table.json", + QE_URL, + "data", + 1700000005000, + "/out/query_engine_data_stats.json", + ) + self.assertEqual( + cmd, + "./target/release/paced_feeder --sink remote-write" + f" --url {QE_URL} --workload-table /out/table.json --metric data" + " --seed 42 --duration-secs 600 --start-ms 1700000005000" + " --stats-out /out/query_engine_data_stats.json", + ) + + def test_optional_flags_are_appended(self): + cmd = paced_feeder_command( + feeder_config("/t", samples_per_sec=20, extra_flags="--concurrency 4"), + "/t", + QE_URL, + "data", + 0, + "/s", + ) + self.assertTrue(cmd.endswith(" --samples-per-sec 20 --concurrency 4")) + + +class PacedFeederSinksTest(unittest.TestCase): + def test_sketchdb_feeds_the_query_engine_only(self): + self.assertEqual( + paced_feeder_sinks({"mode": "sketchdb"}, QE_URL, PROM_URL), + {"query_engine": QE_URL}, + ) + + def test_sketchdb_querying_prometheus_too_feeds_both(self): + self.assertEqual( + paced_feeder_sinks( + {"mode": "sketchdb", "query_prometheus_too": True}, QE_URL, PROM_URL + ), + {"query_engine": QE_URL, "prometheus": PROM_URL}, + ) + + def test_baseline_feeds_prometheus(self): + self.assertEqual( + paced_feeder_sinks({"mode": "baseline"}, QE_URL, PROM_URL), + {"prometheus": PROM_URL}, + ) + + +class CheckPacedFeederConfigTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.table = write_table(self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def test_valid_config(self): + check_paced_feeder_config(feeder_config(self.table), 1, 600) + + def test_unknown_metric(self): + with self.assertRaisesRegex(ValueError, "'cpu' is not in"): + check_paced_feeder_config( + feeder_config(self.table, metrics=["cpu"]), 1, 600 + ) + + def test_records_metric(self): + with self.assertRaisesRegex(ValueError, "type 'records'"): + check_paced_feeder_config( + feeder_config(self.table, metrics=["flows"]), 1, 600 + ) + + def test_sharded_ingest(self): + with self.assertRaisesRegex(ValueError, "parallelism=1"): + check_paced_feeder_config(feeder_config(self.table), 2, 600) + + def test_feeder_ending_before_the_queries(self): + with self.assertRaisesRegex(ValueError, "at least 601"): + check_paced_feeder_config(feeder_config(self.table), 1, 601) + + def test_missing_keys_rejected_with_the_other_exporters(self): + params = OmegaConf.create( + { + "exporters": { + "only_start_if_queries_exist": False, + "exporter_list": {"paced_feeder": {"workload_table": "t"}}, + } + } + ) + config, reason = read_exporter_config(params) + self.assertIsNone(config) + self.assertIn("duration_secs", reason) + + +class RecordingProvider: + def __init__(self, running_count): + self.commands = [] + self.running_count = running_count + + def get_home_dir(self): + return "/home/u" + + def is_remote(self): + return True + + def execute_command(self, node_idx, cmd, **kwargs): + self.commands.append((node_idx, cmd)) + returncode = 0 + if cmd.startswith("pgrep"): + returncode = 0 if cmd.endswith(f"-qx {self.running_count}") else 1 + return subprocess.CompletedProcess(cmd, returncode) + + +class Args: + def get_node_range(self, include_coordinator=True): + return [0, 1, 2] if include_coordinator else [1, 2] + + def get_coordinator_node(self): + return 0 + + +class PacedFeederServiceStartTest(unittest.TestCase): + def start(self, running_count): + provider = RecordingProvider(running_count) + service = PacedFeederService(provider, Args()) + with tempfile.TemporaryDirectory() as local_dir, mock.patch( + "experiment_utils.services.paced_feeder.time.sleep" + ), mock.patch( + "experiment_utils.services.paced_feeder.sync.rsync_dataset_file", + return_value="/exp/paced_feeder_output/table.json", + ): + service.start( + config=feeder_config("/local/table.json"), + sinks={"query_engine": QE_URL, "prometheus": PROM_URL}, + experiment_output_dir="/exp", + local_experiment_dir=local_dir, + ) + return provider.commands + + def test_one_process_per_sink_and_metric_spread_over_workers(self): + commands = self.start(running_count=2) + feeders = [(n, c) for n, c in commands if "--sink" in c] + self.assertEqual( + [ + ( + n, + c.split(" --url ")[1].split()[0], + c.split(" --metric ")[1].split()[0], + ) + for n, c in feeders + ], + [ + (1, QE_URL, "data"), + (2, QE_URL, "data_total"), + (1, PROM_URL, "data"), + (2, PROM_URL, "data_total"), + ], + ) + # One event-time origin, so the sinks get the same timestamps. + self.assertEqual( + len({c.split(" --start-ms ")[1].split()[0] for _, c in feeders}), 1 + ) + self.assertTrue( + all( + "--workload-table /exp/paced_feeder_output/table.json" in c + for _, c in feeders + ) + ) + + def test_dead_feeder_fails_the_experiment(self): + # A wrong URL stops a feeder after its first tick; without this check + # the run would carry on with no data. + with self.assertRaisesRegex(RuntimeError, "expected 2 processes"): + self.start(running_count=1) + + +class StartCommandProvider: + def __init__(self): + self.commands = [] + + def get_home_dir(self): + return "/home/u" + + def execute_command(self, node_idx, cmd, **kwargs): + self.commands.append(cmd) + # netstat finds nothing on the Prometheus port. + return subprocess.CompletedProcess(cmd, 1) + + +class PrometheusRemoteWriteReceiverTest(unittest.TestCase): + def bare_metal_command(self, receiver): + provider = StartCommandProvider() + PrometheusService(provider, 1, 0, remote_write_receiver=receiver).start("/exp") + return provider.commands[-1] + + def test_bare_metal_flag(self): + self.assertIn( + constants.PROMETHEUS_REMOTE_WRITE_RECEIVER_FLAG, + self.bare_metal_command(True), + ) + self.assertNotIn( + constants.PROMETHEUS_REMOTE_WRITE_RECEIVER_FLAG, + self.bare_metal_command(False), + ) + + def test_containerized_flag(self): + provider = StartCommandProvider() + service = DockerPrometheusService(provider, 1, 0, remote_write_receiver=True) + with mock.patch.object(service, "_force_cleanup_container"), mock.patch.object( + service, "_wait_for_service_ready" + ): + service.start("/exp") + self.assertIn( + constants.PROMETHEUS_REMOTE_WRITE_RECEIVER_FLAG, provider.commands[-1] + ) + + def test_enabled_when_the_paced_feeder_is_configured(self): + def service(exporter_list): + cfg = OmegaConf.create( + { + "experiment_params": { + "monitoring": { + "tool": "prometheus", + "deployment_mode": "bare_metal", + }, + "exporters": {"exporter_list": exporter_list}, + } + } + ) + return create_prometheus_service(cfg, StartCommandProvider(), 1, 0) + + self.assertTrue(service({"paced_feeder": {}}).remote_write_receiver) + self.assertFalse(service({"fake_exporter": {}}).remote_write_receiver) + + +class PrometheusForwardingTest(unittest.TestCase): + """In sketchdb mode Prometheus must not forward fed metrics to the query + engine, which the feeder already writes to.""" + + def remote_write_args(self, metrics): + cfg = OmegaConf.create( + { + "streaming": { + "parallelism": 1, + "remote_write": { + "ip": "10.10.1.1", + "base_port": 8080, + "path": "/api/v1/write", + }, + }, + "experiment_params": { + "exporters": { + "only_start_if_queries_exist": False, + "exporter_list": {}, + }, + "query_groups": [], + "metrics": metrics, + }, + } + ) + args = mock.Mock(remote_write_url=None, remote_write_metric_names=None) + with mock.patch("generate_prometheus_config.main"), mock.patch( + "generate_victoriametrics_config.main" + ): + call_generate_prometheus_config( + args, + cfg, + constants.SKETCHDB_EXPERIMENT_NAME, + constants.SKETCHDB_EXPERIMENT_NAME, + ) + return args + + def test_only_fed_metrics_means_no_remote_write(self): + args = self.remote_write_args( + [{"metric": "data", "labels": ["job"], "exporter": "paced_feeder"}] + ) + self.assertIsNone(args.remote_write_url) + + def test_scraped_metrics_are_still_forwarded(self): + args = self.remote_write_args( + [ + {"metric": "data", "labels": ["job"], "exporter": "paced_feeder"}, + {"metric": "node_cpu", "labels": ["job"], "exporter": "node_exporter"}, + ] + ) + self.assertEqual(args.remote_write_url, QE_URL) + self.assertEqual(args.remote_write_metric_names, ["node_cpu"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py b/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py new file mode 100644 index 00000000..f14dd2ba --- /dev/null +++ b/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py @@ -0,0 +1,204 @@ +"""Tests for sketch_bench_e2e_configs.py (e2e configs from a sketch-bench table).""" + +import json +import os +import shutil +import tempfile +import unittest + +from hydra import compose, initialize_config_dir +from omegaconf import OmegaConf + +from experiment_utils import config as experiment_config +from experiment_utils.core import read_exporter_config +from experiment_utils.services.paced_feeder import check_paced_feeder_config +from sketch_bench_e2e import sketch_bench_e2e_configs as sbe +from test_paced_feeder import E2E_METRICS + +CONFIG_DIR = os.path.join(sbe.EXPERIMENTS_DIR, "config") +PREFIX = "synthetic/templates=classic/shared=1" + +# A slice of synthetic-templatesclassic-shared1.json's e2e.queries, plus a +# structured entry from the multi-grouping tables. +QUERIES = [ + { + "interval_secs": 1, + "promql": "sum by (job) (data)", + "rqes": [f"{PREFIX}/sum_by_job/keys/1s"], + }, + { + "interval_secs": 60, + "promql": "sum_over_time(data[15m])", + "rqes": [f"{PREFIX}/sum_over_time/keys/15m"], + }, + { + "interval_secs": 1, + "promql": "quantile by (job) (0.5, data)", + "rqes": [f"{PREFIX}/quantile_by_job_p0.5/values/1s"], + }, + { + "interval_secs": 60, + "promql": "rate(data_total[15m])", + "rqes": [f"{PREFIX}/rate/keys/15m"], + }, + { + "capability": "cardinality", + "grouping": ["dst_subnet"], + "interval_secs": 60, + "rqes": ["synthetic/templates=all/shared=1/t11_distinct_src/keys/5m"], + "table": "flows", + "value": "src_ip", + "window_secs": 300, + }, +] + + +class BuildConfigTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.table = os.path.join(self.tmp.name, "table.json") + with open(self.table, "w") as f: + json.dump( + {"workloads": [{"e2e": {"metrics": E2E_METRICS, "queries": QUERIES}}]}, + f, + ) + + def tearDown(self): + self.tmp.cleanup() + + def build(self, **kwargs): + return sbe.build_config( + self.table, + seed=7, + starting_delay_secs=120, + query_duration_secs=600, + **kwargs, + ) + + def test_query_groups_by_interval_in_table_order(self): + config, _ = self.build() + groups = config["experiment_params"]["query_groups"] + self.assertEqual( + [(g["id"], g["repetition_delay_ms"], g["queries"]) for g in groups], + [ + (1, 1000, ["sum by (job) (data)", "quantile by (job) (0.5, data)"]), + (2, 60000, ["sum_over_time(data[15m])", "rate(data_total[15m])"]), + ], + ) + self.assertEqual( + [g["client_options"] for g in groups], + [ + { + "repetitions": repetitions, + "starting_delay": 120, + "query_time_offset": sbe.DEFAULT_QUERY_TIME_OFFSET_SECS, + "align_to_interval": True, + } + for repetitions in (600, 10) + ], + ) + + def test_feeder_sends_the_remote_writable_metrics(self): + config, notes = self.build() + params = config["experiment_params"] + self.assertEqual( + params["metrics"], + [ + { + "metric": "data", + "labels": ["job", "label_0"], + "exporter": "paced_feeder", + }, + { + "metric": "data_total", + "labels": ["job", "label_0"], + "exporter": "paced_feeder", + }, + ], + ) + self.assertEqual( + params["exporters"]["exporter_list"]["paced_feeder"], + { + "workload_table": self.table, + "metrics": ["data", "data_total"], + "seed": 7, + "duration_secs": 120 + 600 + sbe.FEEDER_SLACK_SECS, + }, + ) + self.assertIn("skipped records metric 'flows': not remote-writable", notes) + + def test_structured_queries_skipped_without_sql(self): + config, notes = self.build() + self.assertEqual(len(config["experiment_params"]["query_groups"]), 2) + self.assertIn( + "skipped 1 structured (multi-grouping) queries: no --sql-queries file", + notes, + ) + + def test_structured_queries_from_sql_file(self): + sql = os.path.join(self.tmp.name, "multigroup.sql") + with open(sql, "w") as f: + f.write("SELECT dst_subnet, uniq(src_ip) FROM flows GROUP BY dst_subnet;\n") + config, notes = self.build(sql_queries_file=sql) + group = config["experiment_params"]["query_groups"][2] + self.assertEqual(group["id"], 3) + self.assertEqual(group["repetition_delay_ms"], 60000) + self.assertEqual( + group["queries"], + ["SELECT dst_subnet, uniq(src_ip) FROM flows GROUP BY dst_subnet"], + ) + self.assertFalse(any("structured" in n for n in notes)) + + def test_sql_file_must_match_the_structured_queries(self): + sql = os.path.join(self.tmp.name, "multigroup.sql") + with open(sql, "w") as f: + f.write("SELECT 1;\nSELECT 2;\n") + with self.assertRaisesRegex(ValueError, "2 statements; the table has 1"): + self.build(sql_queries_file=sql) + + def test_query_phase_shorter_than_an_interval(self): + with self.assertRaisesRegex(ValueError, "shorter than the 60 s"): + sbe.build_config( + self.table, seed=7, starting_delay_secs=0, query_duration_secs=30 + ) + + def test_prebuilt_output_dir(self): + prebuilt = os.path.join(self.tmp.name, "prebuilt") + os.makedirs(prebuilt) + with self.assertRaisesRegex(ValueError, "streaming_config.yaml"): + self.build(prebuilt_output_dir=prebuilt) + for name in experiment_config.PLANNER_OUTPUT_FILES: + open(os.path.join(prebuilt, name), "w").close() + config, _ = self.build(prebuilt_output_dir=prebuilt) + self.assertEqual(config["controller"], {"prebuilt_output_dir": prebuilt}) + + def test_generated_config_composes_and_validates(self): + # A dry run of what experiment_run_e2e.py does with the file before it + # starts any service. + config_dir = os.path.join(self.tmp.name, "config") + shutil.copytree(CONFIG_DIR, config_dir) + sbe.write_config( + self.build()[0], + self.table, + os.path.join(config_dir, "experiment_type", "sketch_bench_e2e", "t.yaml"), + ) + with initialize_config_dir(version_base=None, config_dir=config_dir): + cfg = compose("config", overrides=["experiment_type=sketch_bench_e2e/t"]) + params = cfg.experiment_params + experiment_config.validate_experiment_config(params) + exporter_config, reason = read_exporter_config(params) + self.assertIsNotNone(exporter_config, reason) + check_paced_feeder_config( + OmegaConf.to_container(exporter_config.exporter_list.paced_feeder), + parallelism=cfg.streaming.parallelism, + min_feed_secs=cfg.flow.steady_state_wait + + experiment_config.get_minimum_experiment_running_time(params), + ) + modes, _ = experiment_config.generate_controller_client_configs( + params, self.tmp.name + ) + self.assertEqual(modes, ["sketchdb", "baseline"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/asap-tools/installation/common/setup_dependencies.sh b/asap-tools/installation/common/setup_dependencies.sh index 6120a78a..2e3930a4 100755 --- a/asap-tools/installation/common/setup_dependencies.sh +++ b/asap-tools/installation/common/setup_dependencies.sh @@ -24,6 +24,7 @@ else fi # (cd "$THIS_DIR/../../experiments/fake_exporter_rust/fake_exporter"; cargo build --release) (cd "$THIS_DIR/../../../asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter" && cargo build --release) +(cd "$THIS_DIR/../../../asap-tools/data-sources/paced-feeder" && cargo build --release) DOCKER_DATA_DIR=/scratch/var_lib_docker diff --git a/asap-tools/queriers/prometheus-client/classes/query_time.py b/asap-tools/queriers/prometheus-client/classes/query_time.py new file mode 100644 index 00000000..af208c19 --- /dev/null +++ b/asap-tools/queriers/prometheus-client/classes/query_time.py @@ -0,0 +1,40 @@ +"""Evaluation times aligned to a query group's repeat interval. + +With `client_options.align_to_interval`, repetition k of a group with repeat +interval T (`repetition_delay_ms`) and settle offset O +(`client_options.query_time_offset`, seconds, default 0) evaluates at + + t_k = floor((start - O) / T) * T + k * T + +and is sent at wall-clock time t_k + O. Every evaluation time is a multiple of +T, so all systems answer the same windows (e.g. a 60 s query at whole minutes), +and O gives the data of the window ending at t_k time to arrive first. For +example, T = 60 s and O = 10 s, started at 12:00:35: repetitions evaluate at +12:00:00, 12:01:00, ... and are sent at 12:00:35 (late, so at once), +12:01:10, 12:02:10, ... +""" + +MS_PER_SEC = 1000 + + +def first_aligned_eval_time( + now_secs: float, interval_ms: int, offset_secs: float +) -> int: + """The latest multiple of the interval at least `offset_secs` before now.""" + if interval_ms <= 0 or interval_ms % MS_PER_SEC != 0: + raise ValueError( + f"align_to_interval needs a whole-second repetition_delay_ms, got {interval_ms}" + ) + if offset_secs < 0: + raise ValueError(f"query_time_offset must be >= 0, got {offset_secs}") + interval_secs = interval_ms // MS_PER_SEC + return int((now_secs - offset_secs) // interval_secs) * interval_secs + + +def aligned_eval_time(first_eval_time: int, interval_ms: int, repetition: int) -> int: + return first_eval_time + repetition * (interval_ms // MS_PER_SEC) + + +def secs_until_send(eval_time: int, offset_secs: float, now_secs: float) -> float: + """How long to wait before evaluating at `eval_time`; 0 when already late.""" + return max(0.0, eval_time + offset_secs - now_secs) diff --git a/asap-tools/queriers/prometheus-client/main_prometheus_client.py b/asap-tools/queriers/prometheus-client/main_prometheus_client.py index 104e4d68..15c5e73d 100644 --- a/asap-tools/queriers/prometheus-client/main_prometheus_client.py +++ b/asap-tools/queriers/prometheus-client/main_prometheus_client.py @@ -30,6 +30,11 @@ from classes.query_client import QueryClient from classes.query_client_factory import QueryClientFactory from classes.query_template import QueryTemplate +from classes.query_time import ( + aligned_eval_time, + first_aligned_eval_time, + secs_until_send, +) from promql_utilities.query_results.classes import QueryResult, QueryResultAcrossTime from promql_utilities.query_results.serializers import SerializerFactory @@ -316,20 +321,36 @@ def handle_query_group( for server_name in servers } - for repetition_idx in range(query_group.repetitions): - current_time = datetime.datetime.now() - logger.debug("Current unix time: {}", int(current_time.timestamp())) + # See classes/query_time.py. Aligned groups ignore query_start_times. + align_to_interval = getattr(query_group, "align_to_interval", False) + if align_to_interval: + offset_secs = getattr(query_group, "query_time_offset", 0) + first_eval_time = first_aligned_eval_time( + time.time(), query_group.repetition_delay_ms, offset_secs + ) - if hasattr(query_group, "query_time_offset"): - current_time = current_time - datetime.timedelta( - seconds=query_group.query_time_offset - ) - logger.debug( - "Offsetting query time by {} seconds", query_group.query_time_offset + for repetition_idx in range(query_group.repetitions): + if align_to_interval: + query_unix_time = aligned_eval_time( + first_eval_time, query_group.repetition_delay_ms, repetition_idx ) + time.sleep(secs_until_send(query_unix_time, offset_secs, time.time())) + logger.debug("Aligned evaluation time: {}", query_unix_time) + else: + current_time = datetime.datetime.now() + logger.debug("Current unix time: {}", int(current_time.timestamp())) + + if hasattr(query_group, "query_time_offset"): + current_time = current_time - datetime.timedelta( + seconds=query_group.query_time_offset + ) + logger.debug( + "Offsetting query time by {} seconds", + query_group.query_time_offset, + ) - query_unix_time = int(current_time.timestamp()) - logger.debug("Unix time after query_time_offset: {}", query_unix_time) + query_unix_time = int(current_time.timestamp()) + logger.debug("Unix time after query_time_offset: {}", query_unix_time) if parallel: # Execute queries in parallel @@ -337,11 +358,15 @@ def handle_query_group( futures = [] for local_query_idx, query in enumerate(query_group.queries): global_query_idx = global_query_idx_start + local_query_idx - current_query_unix_time = get_query_unix_time( - query, - query_unix_time, - query_start_times, - query_group.repetition_delay_ms, + current_query_unix_time = ( + query_unix_time + if align_to_interval + else get_query_unix_time( + query, + query_unix_time, + query_start_times, + query_group.repetition_delay_ms, + ) ) for server_name, server_object in servers.items(): @@ -380,11 +405,15 @@ def handle_query_group( # Serial execution - use the same execute_single_query function for local_query_idx, query in enumerate(query_group.queries): global_query_idx = global_query_idx_start + local_query_idx - current_query_unix_time = get_query_unix_time( - query, - query_unix_time, - query_start_times, - query_group.repetition_delay_ms, + current_query_unix_time = ( + query_unix_time + if align_to_interval + else get_query_unix_time( + query, + query_unix_time, + query_start_times, + query_group.repetition_delay_ms, + ) ) logger.debug("Unix time for query: {}", current_query_unix_time) @@ -428,7 +457,8 @@ def handle_query_group( if latency_exporter is not None: latency_exporter.export_repetition(repetition_idx, result) - if repetition_idx < query_group.repetitions - 1: + # Aligned groups wait for their next evaluation time instead. + if not align_to_interval and repetition_idx < query_group.repetitions - 1: time.sleep(query_group.repetition_delay_ms / 1000) if latency_exporter is not None: diff --git a/asap-tools/queriers/prometheus-client/tests/test_query_time.py b/asap-tools/queriers/prometheus-client/tests/test_query_time.py new file mode 100644 index 00000000..3ae121ab --- /dev/null +++ b/asap-tools/queriers/prometheus-client/tests/test_query_time.py @@ -0,0 +1,51 @@ +"""Tests for evaluation times aligned to a query group's repeat interval.""" + +import unittest + +from classes.query_time import ( + aligned_eval_time, + first_aligned_eval_time, + secs_until_send, +) + +# 2026-10-13 12:00:00 UTC, a multiple of 60 s. +MINUTE = 1791892800 + + +class FirstAlignedEvalTimeTest(unittest.TestCase): + def test_floors_to_the_interval_after_the_offset(self) -> None: + # Started at 12:00:35 with a 10 s offset: 12:00:25 floors to 12:00:00. + self.assertEqual(first_aligned_eval_time(MINUTE + 35.4, 60000, 10), MINUTE) + # Started at 12:00:05: the 12:00:00 window has not settled yet. + self.assertEqual(first_aligned_eval_time(MINUTE + 5, 60000, 10), MINUTE - 60) + + def test_one_second_interval(self) -> None: + self.assertEqual(first_aligned_eval_time(MINUTE + 7.9, 1000, 2), MINUTE + 5) + + def test_sub_second_interval_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "whole-second"): + first_aligned_eval_time(MINUTE, 1500, 0) + + def test_negative_offset_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "query_time_offset"): + first_aligned_eval_time(MINUTE, 60000, -1) + + +class ScheduleTest(unittest.TestCase): + def test_repetitions_step_by_the_interval(self) -> None: + self.assertEqual( + [aligned_eval_time(MINUTE, 60000, k) for k in range(3)], + [MINUTE, MINUTE + 60, MINUTE + 120], + ) + + def test_waits_until_offset_after_the_eval_time(self) -> None: + self.assertEqual(secs_until_send(MINUTE + 60, 10, MINUTE + 65), 5) + + def test_late_repetition_runs_at_once(self) -> None: + # A slow repetition makes the next one late; it keeps its eval time + # rather than skipping it. + self.assertEqual(secs_until_send(MINUTE + 60, 10, MINUTE + 75), 0) + + +if __name__ == "__main__": + unittest.main() From f7e5a66c4b98c7f11b4a641474cf3e54b2e71073 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 13 Oct 2026 01:41:26 +0000 Subject: [PATCH 4/9] feat(asap-tools): give Prometheus a 5 s out-of-order window for the paced feeder The feeder sends each tick's requests concurrently, so a series' samples can reach Prometheus' remote-write receiver out of order within a second, and Prometheus rejects them without an out-of-order window. When the paced feeder is configured, the generated prometheus.yml (used by bare-metal and containerized Prometheus) sets storage.tsdb.out_of_order_time_window: 5s. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AnJ7LocWxguh3q4QWkCYae --- .../CONFIG_PARAMETERS_REFERENCE.md | 4 +- asap-tools/experiments/constants.py | 4 ++ .../experiments/generate_prometheus_config.py | 10 +++- .../experiments/tests/test_paced_feeder.py | 52 ++++++++++++++++++- 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md index a85db47e..5050abb1 100644 --- a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md +++ b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md @@ -465,7 +465,9 @@ one process per (sink, metric). It pushes over remote write instead of being scraped: in `sketchdb` mode to the query engine's ingest (`streaming.remote_write`, which must have `parallelism: 1`), plus Prometheus when the mode sets `query_prometheus_too`; in every other mode to Prometheus, -which then starts with `--web.enable-remote-write-receiver`. All processes +which then starts with `--web.enable-remote-write-receiver` and a 5 s +`storage.tsdb.out_of_order_time_window` (a tick's requests are sent +concurrently, so samples can arrive out of order within a second). All processes share the seed and table, so every system gets the same data. Its metrics are listed under `metrics` with `exporter: paced_feeder`; Prometheus does not forward them to the query engine. Generate a whole config from a sketch-bench diff --git a/asap-tools/experiments/constants.py b/asap-tools/experiments/constants.py index 2faf6967..151999af 100644 --- a/asap-tools/experiments/constants.py +++ b/asap-tools/experiments/constants.py @@ -44,6 +44,10 @@ PROMETHEUS_REMOTE_WRITE_ENDPOINT = "/api/v1/write" # Lets Prometheus accept samples pushed to PROMETHEUS_REMOTE_WRITE_ENDPOINT. PROMETHEUS_REMOTE_WRITE_RECEIVER_FLAG = "--web.enable-remote-write-receiver" +# The paced feeder sends a tick's requests concurrently, so a series' samples +# can arrive out of order within that second; Prometheus rejects them without +# an out-of-order window. +PROMETHEUS_OUT_OF_ORDER_TIME_WINDOW = "5s" HTTP_READINESS_CURL_TIMEOUT_SECONDS = 5 QUERY_ENGINE_RUNTIME_INFO_ENDPOINT = "/api/v1/status/runtimeinfo" diff --git a/asap-tools/experiments/generate_prometheus_config.py b/asap-tools/experiments/generate_prometheus_config.py index a988ee72..bcd3beee 100644 --- a/asap-tools/experiments/generate_prometheus_config.py +++ b/asap-tools/experiments/generate_prometheus_config.py @@ -4,7 +4,9 @@ import argparse from omegaconf import DictConfig +import constants import experiment_utils +from experiment_utils.services.paced_feeder import PACED_FEEDER_EXPORTER from utils import get_node_range @@ -158,6 +160,13 @@ def main(args, experiment_config=None): if exporter_config is None: raise ValueError("Invalid exporter config: {}".format(rejection_reason)) + if PACED_FEEDER_EXPORTER in exporter_config["exporter_list"]: + config["storage"] = { + "tsdb": { + "out_of_order_time_window": constants.PROMETHEUS_OUT_OF_ORDER_TIME_WINDOW + } + } + # add section for "scrape_configs" config["scrape_configs"] = [] # add target for self-monitoring prometheus @@ -437,7 +446,6 @@ def main(args, experiment_config=None): config["remote_write"].append(remote_write_config) # Write prometheus config to output_dir - import constants output_file = os.path.join(args.output_dir, constants.PROMETHEUS_CONFIG_FILE) os.makedirs(args.output_dir, exist_ok=True) diff --git a/asap-tools/experiments/tests/test_paced_feeder.py b/asap-tools/experiments/tests/test_paced_feeder.py index 6e55fdc9..2a61c5bb 100644 --- a/asap-tools/experiments/tests/test_paced_feeder.py +++ b/asap-tools/experiments/tests/test_paced_feeder.py @@ -7,10 +7,17 @@ import unittest from unittest import mock +import yaml + from omegaconf import OmegaConf import constants -from experiment_utils.core import call_generate_prometheus_config, read_exporter_config +import generate_prometheus_config +from experiment_utils.core import ( + GeneratePrometheusArgs, + call_generate_prometheus_config, + read_exporter_config, +) from experiment_utils.services import create_prometheus_service from experiment_utils.services.docker_prometheus import DockerPrometheusService from experiment_utils.services.paced_feeder import ( @@ -363,5 +370,48 @@ def test_scraped_metrics_are_still_forwarded(self): self.assertEqual(args.remote_write_metric_names, ["node_cpu"]) +class PrometheusOutOfOrderWindowTest(unittest.TestCase): + """The feeder sends a tick's requests concurrently, so Prometheus' receiver + sees a series' samples out of order within a second and, without an + out-of-order window, rejects them (400 "out of order sample"). Bare-metal + and containerized Prometheus both read this generated prometheus.yml.""" + + def storage(self, exporter_list): + with tempfile.TemporaryDirectory() as out: + prometheus_config = OmegaConf.create( + { + "scrape_interval": "1s", + "evaluation_interval": "10s", + "recording_rules": {"interval": "5s"}, + } + ) + args = GeneratePrometheusArgs( + 1, out, out, prometheus_config, "10.10.1.1", "10.10.1", 0 + ) + generate_prometheus_config.main( + args, + { + "exporters": { + "only_start_if_queries_exist": False, + "exporter_list": exporter_list, + }, + "query_groups": [], + "metrics": [], + }, + ) + with open(os.path.join(out, constants.PROMETHEUS_CONFIG_FILE)) as f: + return yaml.safe_load(f).get("storage") + + def test_window_set_when_the_feeder_targets_prometheus(self): + feeder = {"workload_table": "t", "metrics": [], "seed": 1, "duration_secs": 1} + self.assertEqual( + self.storage({"paced_feeder": feeder}), + {"tsdb": {"out_of_order_time_window": "5s"}}, + ) + + def test_no_window_without_the_feeder(self): + self.assertIsNone(self.storage({})) + + if __name__ == "__main__": unittest.main() From 0c6434a1d09990821cb9a49472ba43c48dfdbbda Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 13 Oct 2026 01:52:13 +0000 Subject: [PATCH 5/9] feat(asap-tools): feed records metrics to ASAPQuery only Records metrics (http, flows) reach remote write as one sample per value field, many sharing a series and timestamp, which Prometheus' receiver rejects. check_paced_feeder_config now takes the experiment modes: it accepts records metrics when every mode feeds ASAPQuery only and refuses them when a mode feeds Prometheus (baseline or query_prometheus_too). The config generator feeds a table's records metrics only with --sql-queries (only the structured queries read them); the config then has no baseline mode, and records metrics stay out of `metrics`, which with prebuilt planner configs only keeps fed metrics out of Prometheus' forwarding. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AnJ7LocWxguh3q4QWkCYae --- .../CONFIG_PARAMETERS_REFERENCE.md | 2 +- asap-tools/experiments/experiment_run_e2e.py | 9 ++-- .../experiment_utils/services/paced_feeder.py | 45 ++++++++++++++----- .../sketch_bench_e2e_configs.py | 42 ++++++++++++----- .../experiments/tests/test_paced_feeder.py | 36 ++++++++++++--- .../tests/test_sketch_bench_e2e_configs.py | 44 +++++++++++++++--- 6 files changed, 137 insertions(+), 41 deletions(-) diff --git a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md index 5050abb1..55800114 100644 --- a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md +++ b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md @@ -478,7 +478,7 @@ table with `sketch_bench_e2e/sketch_bench_e2e_configs.py`. - **Example**: `/path/to/synthetic-templatesclassic-shared1.json` #### `experiment_params.exporters.exporter_list.paced_feeder.metrics` (list, required) -- **Description**: Metrics of the table's `workloads[0].e2e.metrics` to send; gauge or counter only +- **Description**: Metrics of the table's `workloads[0].e2e.metrics` to send. Gauge and counter metrics go to every sink; `records` metrics (one sample per value field, `.`) only to ASAPQuery, so they are refused when any mode feeds Prometheus (baseline, or `query_prometheus_too`) - **Example**: `[data, data_total]` #### `experiment_params.exporters.exporter_list.paced_feeder.seed` (int, required) diff --git a/asap-tools/experiments/experiment_run_e2e.py b/asap-tools/experiments/experiment_run_e2e.py index f1a6ef1a..805c65c9 100644 --- a/asap-tools/experiments/experiment_run_e2e.py +++ b/asap-tools/experiments/experiment_run_e2e.py @@ -192,21 +192,22 @@ def main(cfg: DictConfig): cfg.experiment_params ) + modes_by_name = { + m["mode"]: m + for m in OmegaConf.to_container(cfg.experiment_params.experiment, resolve=True) + } paced_feeder_service = None paced_feeder_config = exporter_config["exporter_list"].get(PACED_FEEDER_EXPORTER) if paced_feeder_config is not None: paced_feeder_config = OmegaConf.to_container(paced_feeder_config, resolve=True) check_paced_feeder_config( paced_feeder_config, + modes=list(modes_by_name.values()), parallelism=args.parallelism, min_feed_secs=(0 if skip_querying else args.steady_state_wait) + minimum_experiment_running_time, ) paced_feeder_service = PacedFeederService(provider, args) - modes_by_name = { - m["mode"]: m - for m in OmegaConf.to_container(cfg.experiment_params.experiment, resolve=True) - } for experiment_mode in experiment_modes: print(f"Running experiment mode: {experiment_mode}") diff --git a/asap-tools/experiments/experiment_utils/services/paced_feeder.py b/asap-tools/experiments/experiment_utils/services/paced_feeder.py index c01abb3e..40a31ece 100644 --- a/asap-tools/experiments/experiment_utils/services/paced_feeder.py +++ b/asap-tools/experiments/experiment_utils/services/paced_feeder.py @@ -21,8 +21,11 @@ PACED_FEEDER_EXPORTER = "paced_feeder" PACED_FEEDER_REQUIRED_KEYS = ("workload_table", "metrics", "seed", "duration_secs") -# Metric types a remote-write sink (ASAPQuery ingest, Prometheus) accepts. -REMOTE_WRITE_METRIC_TYPES = ("gauge", "counter") +# Metric types Prometheus' remote-write receiver accepts. +PROMETHEUS_METRIC_TYPES = ("gauge", "counter") +# Streams of records, sent as one sample per value field (`http.latency`). +# Many records share a series and timestamp, so only ASAPQuery takes them. +RECORDS_METRIC_TYPE = "records" PACED_FEEDER_BINARY = "./target/release/paced_feeder" # The brackets keep pgrep/pkill from matching the shell that runs them. @@ -47,11 +50,23 @@ def paced_feeder_metric_names(experiment_params) -> List[str]: ] +def feeds_prometheus(mode: Dict[str, Any]) -> bool: + """Whether the feeder sends to Prometheus in this experiment mode.""" + return mode["mode"] != constants.SKETCHDB_EXPERIMENT_NAME or mode.get( + "query_prometheus_too", False + ) + + def check_paced_feeder_config( - config: Dict[str, Any], parallelism: int, min_feed_secs: float + config: Dict[str, Any], + modes: List[Dict[str, Any]], + parallelism: int, + min_feed_secs: float, ) -> None: - """Fail before the experiment starts on a config the feeder would reject - or one that stops sending before the queries end.""" + """Fail before the experiment starts on a config the feeder would reject, + one sending records to Prometheus, or one that stops sending before the + queries end.""" + prometheus_modes = [m["mode"] for m in modes if feeds_prometheus(m)] with open(config["workload_table"]) as f: (workload,) = json.load(f)["workloads"] table_metrics = workload["e2e"]["metrics"] @@ -62,10 +77,16 @@ def check_paced_feeder_config( f"{config['workload_table']} (has {sorted(table_metrics)})" ) metric_type = table_metrics[metric]["type"] - if metric_type not in REMOTE_WRITE_METRIC_TYPES: + if metric_type == RECORDS_METRIC_TYPE: + if prometheus_modes: + raise ValueError( + f"paced_feeder metric {metric!r} is a records metric, which " + f"only ASAPQuery takes, but modes {prometheus_modes} feed " + "Prometheus (baseline or query_prometheus_too)" + ) + elif metric_type not in PROMETHEUS_METRIC_TYPES: raise ValueError( - f"paced_feeder metric {metric!r} has type {metric_type!r}; " - f"remote write takes only {REMOTE_WRITE_METRIC_TYPES}" + f"paced_feeder metric {metric!r} has unknown type {metric_type!r}" ) if parallelism != 1: raise ValueError( @@ -87,10 +108,10 @@ def paced_feeder_sinks( sketchdb feeds ASAPQuery's ingest, plus Prometheus when the mode also queries it; every other mode queries Prometheus only. """ - if mode["mode"] != constants.SKETCHDB_EXPERIMENT_NAME: - return {SINK_PROMETHEUS: prometheus_url} - sinks = {SINK_QUERY_ENGINE: query_engine_url} - if mode.get("query_prometheus_too", False): + sinks = {} + if mode["mode"] == constants.SKETCHDB_EXPERIMENT_NAME: + sinks[SINK_QUERY_ENGINE] = query_engine_url + if feeds_prometheus(mode): sinks[SINK_PROMETHEUS] = prometheus_url return sinks diff --git a/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py b/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py index 27a5eb9d..f1b2775c 100644 --- a/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py +++ b/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py @@ -10,8 +10,12 @@ groups, only when `--sql-queries` gives their SQL; otherwise they are skipped with a note. The file holds one `;`-terminated statement per structured entry, in table order. -- the table's gauge and counter metrics, sent by the paced feeder (records - metrics are skipped with a note: remote write cannot carry them). +- the table's gauge and counter metrics, sent by the paced feeder and listed + under `metrics`. Records metrics, which only the structured queries read, + are fed only with `--sql-queries`, and then only to ASAPQuery: the config + has no baseline (Prometheus) mode, and they are not listed under `metrics` + (with prebuilt planner configs that list only keeps fed metrics out of + Prometheus' forwarding, and records are never scraped). - `controller.prebuilt_output_dir`, when given, so ASAPQuery runs those planner configs (e.g. from `asap-planner --planner saved`). @@ -41,7 +45,8 @@ ) from experiment_utils.services.paced_feeder import ( # noqa: E402 PACED_FEEDER_EXPORTER, - REMOTE_WRITE_METRIC_TYPES, + PROMETHEUS_METRIC_TYPES, + RECORDS_METRIC_TYPE, ) DEFAULT_OUTPUT_DIR = os.path.join( @@ -51,10 +56,8 @@ {"name": "prometheus", "url": "http://localhost:9090"}, {"name": "sketchdb", "url": "http://localhost:8088"}, ] -MODES = [ - {"mode": "sketchdb", "server": "sketchdb"}, - {"mode": "baseline", "server": "prometheus"}, -] +SKETCHDB_MODE = {"mode": "sketchdb", "server": "sketchdb"} +BASELINE_MODE = {"mode": "baseline", "server": "prometheus"} # Seconds the window ending at an evaluation time gets for its data to arrive. DEFAULT_QUERY_TIME_OFFSET_SECS = 10 # Feeder time beyond the query phase: covers flow.steady_state_wait (60 s in @@ -134,10 +137,18 @@ def build_config( raise ValueError(f"{table_path}: no queries to run") metrics = [] + records_metrics = [] for name, desc in e2e["metrics"].items(): - if desc["type"] not in REMOTE_WRITE_METRIC_TYPES: - notes.append(f"skipped {desc['type']} metric {name!r}: not remote-writable") + if desc["type"] == RECORDS_METRIC_TYPE: + if sql: + records_metrics.append(name) + else: + notes.append( + f"skipped records metric {name!r}: only structured queries read it" + ) continue + if desc["type"] not in PROMETHEUS_METRIC_TYPES: + raise ValueError(f"metric {name!r} has unknown type {desc['type']!r}") metrics.append( { "metric": name, @@ -148,7 +159,7 @@ def build_config( feeder = { "workload_table": os.path.abspath(table_path), - "metrics": [m["metric"] for m in metrics], + "metrics": [m["metric"] for m in metrics] + records_metrics, "seed": seed, "duration_secs": ( feeder_duration_secs @@ -159,9 +170,18 @@ def build_config( if samples_per_sec is not None: feeder["samples_per_sec"] = samples_per_sec + modes = [SKETCHDB_MODE] + if records_metrics: + notes.append( + "no baseline mode: records metrics " + f"{records_metrics} go to ASAPQuery only" + ) + else: + modes.append(BASELINE_MODE) + config: dict = { "experiment_params": { - "experiment": MODES, + "experiment": modes, "monitoring": {"tool": "prometheus", "deployment_mode": "bare_metal"}, "servers": SERVERS, "exporters": { diff --git a/asap-tools/experiments/tests/test_paced_feeder.py b/asap-tools/experiments/tests/test_paced_feeder.py index 2a61c5bb..e2aa05e9 100644 --- a/asap-tools/experiments/tests/test_paced_feeder.py +++ b/asap-tools/experiments/tests/test_paced_feeder.py @@ -29,6 +29,9 @@ from experiment_utils.services.prometheus import PrometheusService QE_URL = "http://10.10.1.1:8080/api/v1/write" +SKETCHDB = {"mode": "sketchdb", "server": "sketchdb"} +BASELINE = {"mode": "baseline", "server": "prometheus"} +BOTH_MODES = [SKETCHDB, BASELINE] PROM_URL = "http://10.10.1.1:9090/api/v1/write" # workloads[0].e2e.metrics of synthetic-templatesclassic-shared1.json, plus a @@ -139,27 +142,46 @@ def tearDown(self): self.tmp.cleanup() def test_valid_config(self): - check_paced_feeder_config(feeder_config(self.table), 1, 600) + check_paced_feeder_config(feeder_config(self.table), BOTH_MODES, 1, 600) def test_unknown_metric(self): with self.assertRaisesRegex(ValueError, "'cpu' is not in"): check_paced_feeder_config( - feeder_config(self.table, metrics=["cpu"]), 1, 600 + feeder_config(self.table, metrics=["cpu"]), BOTH_MODES, 1, 600 ) - def test_records_metric(self): - with self.assertRaisesRegex(ValueError, "type 'records'"): + def test_records_metric_to_query_engine_only(self): + check_paced_feeder_config( + feeder_config(self.table, metrics=["data", "flows"]), [SKETCHDB], 1, 600 + ) + + def test_records_metric_refused_for_prometheus(self): + # Many records share a series and timestamp; Prometheus' receiver + # rejects those duplicates. + records = feeder_config(self.table, metrics=["flows"]) + with self.assertRaisesRegex(ValueError, r"records metric.*\['baseline'\]"): + check_paced_feeder_config(records, BOTH_MODES, 1, 600) + with self.assertRaisesRegex(ValueError, r"query_prometheus_too"): check_paced_feeder_config( - feeder_config(self.table, metrics=["flows"]), 1, 600 + records, [{**SKETCHDB, "query_prometheus_too": True}], 1, 600 ) + def test_unknown_metric_type(self): + with open(self.table) as f: + table = json.load(f) + table["workloads"][0]["e2e"]["metrics"]["data"]["type"] = "histogram" + with open(self.table, "w") as f: + json.dump(table, f) + with self.assertRaisesRegex(ValueError, "unknown type 'histogram'"): + check_paced_feeder_config(feeder_config(self.table), [SKETCHDB], 1, 600) + def test_sharded_ingest(self): with self.assertRaisesRegex(ValueError, "parallelism=1"): - check_paced_feeder_config(feeder_config(self.table), 2, 600) + check_paced_feeder_config(feeder_config(self.table), BOTH_MODES, 2, 600) def test_feeder_ending_before_the_queries(self): with self.assertRaisesRegex(ValueError, "at least 601"): - check_paced_feeder_config(feeder_config(self.table), 1, 601) + check_paced_feeder_config(feeder_config(self.table), BOTH_MODES, 1, 601) def test_missing_keys_rejected_with_the_other_exporters(self): params = OmegaConf.create( diff --git a/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py b/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py index f14dd2ba..1b31bc9d 100644 --- a/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py +++ b/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py @@ -98,7 +98,7 @@ def test_query_groups_by_interval_in_table_order(self): ], ) - def test_feeder_sends_the_remote_writable_metrics(self): + def test_feeder_sends_the_classic_metrics_without_sql(self): config, notes = self.build() params = config["experiment_params"] self.assertEqual( @@ -125,7 +125,12 @@ def test_feeder_sends_the_remote_writable_metrics(self): "duration_secs": 120 + 600 + sbe.FEEDER_SLACK_SECS, }, ) - self.assertIn("skipped records metric 'flows': not remote-writable", notes) + self.assertIn( + "skipped records metric 'flows': only structured queries read it", notes + ) + self.assertEqual( + [m["mode"] for m in params["experiment"]], ["sketchdb", "baseline"] + ) def test_structured_queries_skipped_without_sql(self): config, notes = self.build() @@ -148,6 +153,20 @@ def test_structured_queries_from_sql_file(self): ["SELECT dst_subnet, uniq(src_ip) FROM flows GROUP BY dst_subnet"], ) self.assertFalse(any("structured" in n for n in notes)) + # The records metric goes to ASAPQuery only: fed, but no baseline + # mode and not under `metrics`. + params = config["experiment_params"] + self.assertEqual( + params["exporters"]["exporter_list"]["paced_feeder"]["metrics"], + ["data", "data_total", "flows"], + ) + self.assertEqual([m["mode"] for m in params["experiment"]], ["sketchdb"]) + self.assertEqual( + [m["metric"] for m in params["metrics"]], ["data", "data_total"] + ) + self.assertIn( + "no baseline mode: records metrics ['flows'] go to ASAPQuery only", notes + ) def test_sql_file_must_match_the_structured_queries(self): sql = os.path.join(self.tmp.name, "multigroup.sql") @@ -172,13 +191,13 @@ def test_prebuilt_output_dir(self): config, _ = self.build(prebuilt_output_dir=prebuilt) self.assertEqual(config["controller"], {"prebuilt_output_dir": prebuilt}) - def test_generated_config_composes_and_validates(self): + def compose_and_validate(self, config): # A dry run of what experiment_run_e2e.py does with the file before it - # starts any service. + # starts any service; returns the experiment modes. config_dir = os.path.join(self.tmp.name, "config") shutil.copytree(CONFIG_DIR, config_dir) sbe.write_config( - self.build()[0], + config, self.table, os.path.join(config_dir, "experiment_type", "sketch_bench_e2e", "t.yaml"), ) @@ -190,6 +209,7 @@ def test_generated_config_composes_and_validates(self): self.assertIsNotNone(exporter_config, reason) check_paced_feeder_config( OmegaConf.to_container(exporter_config.exporter_list.paced_feeder), + modes=OmegaConf.to_container(params.experiment), parallelism=cfg.streaming.parallelism, min_feed_secs=cfg.flow.steady_state_wait + experiment_config.get_minimum_experiment_running_time(params), @@ -197,7 +217,19 @@ def test_generated_config_composes_and_validates(self): modes, _ = experiment_config.generate_controller_client_configs( params, self.tmp.name ) - self.assertEqual(modes, ["sketchdb", "baseline"]) + return modes + + def test_generated_config_composes_and_validates(self): + self.assertEqual( + self.compose_and_validate(self.build()[0]), ["sketchdb", "baseline"] + ) + + def test_generated_config_with_records_validates(self): + sql = os.path.join(self.tmp.name, "multigroup.sql") + with open(sql, "w") as f: + f.write("SELECT 1;\n") + config, _ = self.build(sql_queries_file=sql) + self.assertEqual(self.compose_and_validate(config), ["sketchdb"]) if __name__ == "__main__": From 464edb6d5508d5e6e4b9bd9d28b70b7402ce6607 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 13 Oct 2026 02:02:13 +0000 Subject: [PATCH 6/9] fix(asap-tools): address review on the paced feeder runner - Pass --samples-per-sec only to gauge and counter feeders and the new records_per_sec as --records-per-sec only to records feeders; the feeder exits on the other kind's flag. - SQL query groups get time_window_seconds: 0; their templates read only {{ end_time }} / {{ end_datetime }}, but the client refuses to render templates without a window. - Take the shared --start-ms after the per-node binary check and table copy, so the 5 s lead covers only the launches. - duration_secs must also cover the feeder's startup check before steady_state_wait and the queries. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AnJ7LocWxguh3q4QWkCYae --- .../CONFIG_PARAMETERS_REFERENCE.md | 8 +- asap-tools/experiments/experiment_run_e2e.py | 7 +- .../experiment_utils/services/__init__.py | 2 + .../experiment_utils/services/paced_feeder.py | 41 +++++++-- .../sketch_bench_e2e_configs.py | 13 ++- .../experiments/tests/test_paced_feeder.py | 90 ++++++++++++++++++- .../tests/test_sketch_bench_e2e_configs.py | 17 ++++ 7 files changed, 160 insertions(+), 18 deletions(-) diff --git a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md index 55800114..dbbef000 100644 --- a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md +++ b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md @@ -486,13 +486,17 @@ table with `sketch_bench_e2e/sketch_bench_e2e_configs.py`. - **Example**: `42` #### `experiment_params.exporters.exporter_list.paced_feeder.duration_secs` (int, required) -- **Description**: Seconds of data to send. Must be at least `flow.steady_state_wait` plus the query groups' running time +- **Description**: Seconds of data to send. Must be at least the feeder's startup check (7 s) plus `flow.steady_state_wait` plus the query groups' running time - **Example**: `4000` #### `experiment_params.exporters.exporter_list.paced_feeder.samples_per_sec` (int, optional) -- **Description**: Overrides the table's `samples_per_sec_per_series` +- **Description**: Overrides the table's `samples_per_sec_per_series` (gauge and counter metrics) - **Example**: `20` +#### `experiment_params.exporters.exporter_list.paced_feeder.records_per_sec` (int, optional) +- **Description**: Overrides the table's `records_per_sec` (records metrics) +- **Example**: `20000` + #### `experiment_params.exporters.exporter_list.paced_feeder.extra_flags` (string, optional) - **Description**: Appended to every feeder command line - **Example**: `"--concurrency 32"` diff --git a/asap-tools/experiments/experiment_run_e2e.py b/asap-tools/experiments/experiment_run_e2e.py index 805c65c9..bdf5268e 100644 --- a/asap-tools/experiments/experiment_run_e2e.py +++ b/asap-tools/experiments/experiment_run_e2e.py @@ -28,6 +28,7 @@ PACED_FEEDER_EXPORTER, PacedFeederService, check_paced_feeder_config, + paced_feeder_min_duration_secs, paced_feeder_sinks, ) from experiment_utils.services.misc import DiscoveryBackend @@ -204,8 +205,10 @@ def main(cfg: DictConfig): paced_feeder_config, modes=list(modes_by_name.values()), parallelism=args.parallelism, - min_feed_secs=(0 if skip_querying else args.steady_state_wait) - + minimum_experiment_running_time, + min_feed_secs=paced_feeder_min_duration_secs( + 0 if skip_querying else args.steady_state_wait, + minimum_experiment_running_time, + ), ) paced_feeder_service = PacedFeederService(provider, args) diff --git a/asap-tools/experiments/experiment_utils/services/__init__.py b/asap-tools/experiments/experiment_utils/services/__init__.py index e4581dbb..32cd0d53 100644 --- a/asap-tools/experiments/experiment_utils/services/__init__.py +++ b/asap-tools/experiments/experiment_utils/services/__init__.py @@ -38,6 +38,7 @@ PACED_FEEDER_EXPORTER, PacedFeederService, check_paced_feeder_config, + paced_feeder_min_duration_secs, paced_feeder_sinks, ) from .clickhouse_service import ClickHouseService, ClickHouseDataLoaderService @@ -164,5 +165,6 @@ def create_prometheus_service(cfg, provider, num_nodes: int, node_offset: int): "PACED_FEEDER_EXPORTER", "PacedFeederService", "check_paced_feeder_config", + "paced_feeder_min_duration_secs", "paced_feeder_sinks", ] diff --git a/asap-tools/experiments/experiment_utils/services/paced_feeder.py b/asap-tools/experiments/experiment_utils/services/paced_feeder.py index 40a31ece..94095324 100644 --- a/asap-tools/experiments/experiment_utils/services/paced_feeder.py +++ b/asap-tools/experiments/experiment_utils/services/paced_feeder.py @@ -57,6 +57,23 @@ def feeds_prometheus(mode: Dict[str, Any]) -> bool: ) +def paced_feeder_min_duration_secs( + steady_state_wait: float, query_running_secs: float +) -> float: + """Seconds of data that last until the final query: queries end at most + startup check + steady_state_wait + the query groups' running time after + launch, and the data starts PACED_FEEDER_START_LEAD_SECS after launch (not + subtracted, for slack).""" + return PACED_FEEDER_STARTUP_CHECK_SECS + steady_state_wait + query_running_secs + + +def table_metric_types(workload_table: str) -> Dict[str, str]: + """Metric name -> type in the table's `workloads[0].e2e.metrics`.""" + with open(workload_table) as f: + (workload,) = json.load(f)["workloads"] + return {name: m["type"] for name, m in workload["e2e"]["metrics"].items()} + + def check_paced_feeder_config( config: Dict[str, Any], modes: List[Dict[str, Any]], @@ -67,16 +84,14 @@ def check_paced_feeder_config( one sending records to Prometheus, or one that stops sending before the queries end.""" prometheus_modes = [m["mode"] for m in modes if feeds_prometheus(m)] - with open(config["workload_table"]) as f: - (workload,) = json.load(f)["workloads"] - table_metrics = workload["e2e"]["metrics"] + table_metrics = table_metric_types(config["workload_table"]) for metric in config["metrics"]: if metric not in table_metrics: raise ValueError( f"paced_feeder metric {metric!r} is not in " f"{config['workload_table']} (has {sorted(table_metrics)})" ) - metric_type = table_metrics[metric]["type"] + metric_type = table_metrics[metric] if metric_type == RECORDS_METRIC_TYPE: if prometheus_modes: raise ValueError( @@ -96,7 +111,8 @@ def check_paced_feeder_config( raise ValueError( f"paced_feeder.duration_secs={config['duration_secs']} ends before the " f"queries do: needs at least {min_feed_secs} " - "(flow.steady_state_wait + the query groups' running time)" + "(feeder startup + flow.steady_state_wait + the query groups' " + "running time)" ) @@ -121,6 +137,7 @@ def paced_feeder_command( workload_table: str, url: str, metric: str, + metric_type: str, start_ms: int, stats_out: str, ) -> str: @@ -130,7 +147,11 @@ def paced_feeder_command( f" --seed {config['seed']} --duration-secs {config['duration_secs']}" f" --start-ms {start_ms} --stats-out {stats_out}" ) - if config.get("samples_per_sec") is not None: + # The feeder refuses the other kind's rate flag. + if metric_type == RECORDS_METRIC_TYPE: + if config.get("records_per_sec") is not None: + cmd += f" --records-per-sec {config['records_per_sec']}" + elif config.get("samples_per_sec") is not None: cmd += f" --samples-per-sec {config['samples_per_sec']}" if config.get("extra_flags"): cmd += f" {config['extra_flags']}" @@ -161,9 +182,7 @@ def start( **kwargs, ) -> None: output_dir = os.path.join(experiment_output_dir, "paced_feeder_output") - # One shared event-time origin for every process of this mode. - start_ms = (math.ceil(time.time()) + PACED_FEEDER_START_LEAD_SECS) * 1000 - + metric_types = table_metric_types(config["workload_table"]) jobs = [(sink, metric) for sink in sinks for metric in config["metrics"]] job_nodes = [ self.target_nodes[i % len(self.target_nodes)] for i in range(len(jobs)) @@ -184,6 +203,9 @@ def start( self.provider, config["workload_table"], output_dir, node_idx ) + # One shared event-time origin for every process of this mode, taken + # after the SSH prep above so the lead covers only the launches. + start_ms = (math.ceil(time.time()) + PACED_FEEDER_START_LEAD_SECS) * 1000 node_cmds: List[Tuple[int, str]] = [] for node_idx, (sink, metric) in zip(job_nodes, jobs): name = f"{sink}_{metric}" @@ -192,6 +214,7 @@ def start( workload_table, sinks[sink], metric, + metric_types[metric], start_ms, os.path.join(output_dir, f"{name}_stats.json"), ) diff --git a/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py b/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py index f1b2775c..dd7bee6a 100644 --- a/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py +++ b/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py @@ -86,6 +86,7 @@ def build_config( query_time_offset_secs: int = DEFAULT_QUERY_TIME_OFFSET_SECS, feeder_duration_secs: Optional[int] = None, samples_per_sec: Optional[int] = None, + records_per_sec: Optional[int] = None, prebuilt_output_dir: Optional[str] = None, sql_queries_file: Optional[str] = None, ) -> Tuple[dict, List[str]]: @@ -113,7 +114,12 @@ def build_config( sql = [(q["interval_secs"], s) for q, s in zip(structured, statements)] query_groups = [] - for queries_by_interval in (by_interval(promql), by_interval(sql)): + for queries_by_interval, extra in ( + (by_interval(promql), {}), + # The SQL templates read only {{ end_time }} / {{ end_datetime }} (the + # window is in the SQL); the client requires a window to render them. + (by_interval(sql), {"time_window_seconds": 0}), + ): for interval_secs, queries in queries_by_interval.items(): if query_duration_secs < interval_secs: raise ValueError( @@ -131,6 +137,7 @@ def build_config( "query_time_offset": query_time_offset_secs, "align_to_interval": True, }, + **extra, } ) if not query_groups: @@ -169,6 +176,8 @@ def build_config( } if samples_per_sec is not None: feeder["samples_per_sec"] = samples_per_sec + if records_per_sec is not None and records_metrics: + feeder["records_per_sec"] = records_per_sec modes = [SKETCHDB_MODE] if records_metrics: @@ -231,6 +240,7 @@ def main() -> None: help=f"default: starting delay + query duration + {FEEDER_SLACK_SECS}", ) parser.add_argument("--samples-per-sec", type=int) + parser.add_argument("--records-per-sec", type=int) parser.add_argument("--prebuilt-output-dir") parser.add_argument("--sql-queries") parser.add_argument("--output", help="default: /
.yaml") @@ -244,6 +254,7 @@ def main() -> None: query_time_offset_secs=args.query_time_offset_secs, feeder_duration_secs=args.feeder_duration_secs, samples_per_sec=args.samples_per_sec, + records_per_sec=args.records_per_sec, prebuilt_output_dir=args.prebuilt_output_dir, sql_queries_file=args.sql_queries, ) diff --git a/asap-tools/experiments/tests/test_paced_feeder.py b/asap-tools/experiments/tests/test_paced_feeder.py index e2aa05e9..1484a80b 100644 --- a/asap-tools/experiments/tests/test_paced_feeder.py +++ b/asap-tools/experiments/tests/test_paced_feeder.py @@ -21,8 +21,11 @@ from experiment_utils.services import create_prometheus_service from experiment_utils.services.docker_prometheus import DockerPrometheusService from experiment_utils.services.paced_feeder import ( + PACED_FEEDER_START_LEAD_SECS, + PACED_FEEDER_STARTUP_CHECK_SECS, PacedFeederService, check_paced_feeder_config, + paced_feeder_min_duration_secs, paced_feeder_command, paced_feeder_sinks, ) @@ -88,6 +91,7 @@ def test_command_line(self): "/out/table.json", QE_URL, "data", + "gauge", 1700000005000, "/out/query_engine_data_stats.json", ) @@ -105,11 +109,28 @@ def test_optional_flags_are_appended(self): "/t", QE_URL, "data", + "gauge", 0, "/s", ) self.assertTrue(cmd.endswith(" --samples-per-sec 20 --concurrency 4")) + def test_each_metric_gets_its_own_kind_of_rate(self): + # The feeder exits on --samples-per-sec for a records metric (and on + # --records-per-sec for a gauge), which the startup check would only + # report as a dead process. + config = feeder_config("/t", samples_per_sec=20, records_per_sec=1000) + + def rate_flags(metric, metric_type): + cmd = paced_feeder_command( + config, "/t", QE_URL, metric, metric_type, 0, "/s" + ) + return [f for f in cmd.split() if f.endswith("-per-sec")] + + self.assertEqual(rate_flags("data", "gauge"), ["--samples-per-sec"]) + self.assertEqual(rate_flags("data_total", "counter"), ["--samples-per-sec"]) + self.assertEqual(rate_flags("flows", "records"), ["--records-per-sec"]) + class PacedFeederSinksTest(unittest.TestCase): def test_sketchdb_feeds_the_query_engine_only(self): @@ -183,6 +204,18 @@ def test_feeder_ending_before_the_queries(self): with self.assertRaisesRegex(ValueError, "at least 601"): check_paced_feeder_config(feeder_config(self.table), BOTH_MODES, 1, 601) + def test_duration_covers_startup_wait_and_queries(self): + # 60 s steady_state_wait and 533 s of queries: the feeder's startup + # check comes on top, so 600 s is exactly enough and 599 s is not. + min_secs = paced_feeder_min_duration_secs(60, 533) + self.assertEqual(min_secs, PACED_FEEDER_STARTUP_CHECK_SECS + 60 + 533) + self.assertEqual(min_secs, 600) + check_paced_feeder_config(feeder_config(self.table), BOTH_MODES, 1, min_secs) + with self.assertRaisesRegex(ValueError, "at least 600"): + check_paced_feeder_config( + feeder_config(self.table, duration_secs=599), BOTH_MODES, 1, min_secs + ) + def test_missing_keys_rejected_with_the_other_exporters(self): params = OmegaConf.create( { @@ -197,10 +230,15 @@ def test_missing_keys_rejected_with_the_other_exporters(self): self.assertIn("duration_secs", reason) +# Fake seconds each SSH prep step (binary check, table copy) takes. +PREP_SECS = 10 + + class RecordingProvider: def __init__(self, running_count): self.commands = [] self.running_count = running_count + self.clock = 1700000000.0 def get_home_dir(self): return "/home/u" @@ -210,6 +248,8 @@ def is_remote(self): def execute_command(self, node_idx, cmd, **kwargs): self.commands.append((node_idx, cmd)) + if cmd.startswith("test -x"): + self.clock += PREP_SECS returncode = 0 if cmd.startswith("pgrep"): returncode = 0 if cmd.endswith(f"-qx {self.running_count}") else 1 @@ -225,23 +265,65 @@ def get_coordinator_node(self): class PacedFeederServiceStartTest(unittest.TestCase): - def start(self, running_count): + def start(self, running_count, sinks=None, **config_overrides): provider = RecordingProvider(running_count) + self.provider = provider service = PacedFeederService(provider, Args()) + + def rsync(*args): + provider.clock += PREP_SECS + return "/exp/paced_feeder_output/table.json" + with tempfile.TemporaryDirectory() as local_dir, mock.patch( "experiment_utils.services.paced_feeder.time.sleep" + ), mock.patch( + "experiment_utils.services.paced_feeder.time.time", + lambda: provider.clock, ), mock.patch( "experiment_utils.services.paced_feeder.sync.rsync_dataset_file", - return_value="/exp/paced_feeder_output/table.json", + side_effect=rsync, ): service.start( - config=feeder_config("/local/table.json"), - sinks={"query_engine": QE_URL, "prometheus": PROM_URL}, + config=feeder_config(write_table(local_dir), **config_overrides), + sinks=sinks or {"query_engine": QE_URL, "prometheus": PROM_URL}, experiment_output_dir="/exp", local_experiment_dir=local_dir, ) return provider.commands + def test_start_time_leads_the_launches_not_the_prep(self): + # The SSH prep (2 nodes x binary check + table copy) takes 40 fake + # seconds; the shared start must still be the lead after it. + commands = self.start(running_count=2) + start_secs = { + int(c.split(" --start-ms ")[1].split()[0]) / 1000 + for _, c in commands + if "--sink" in c + } + self.assertEqual( + start_secs, {1700000000 + 4 * PREP_SECS + PACED_FEEDER_START_LEAD_SECS} + ) + + def test_records_and_gauge_processes_get_their_rate_flags(self): + commands = self.start( + running_count=1, + sinks={"query_engine": QE_URL}, + metrics=["data", "flows"], + samples_per_sec=20, + records_per_sec=1000, + ) + rate_flags = { + c.split(" --metric ")[1].split()[0]: [ + f for f in c.split() if f.endswith("-per-sec") + ] + for _, c in commands + if "--sink" in c + } + self.assertEqual( + rate_flags, + {"data": ["--samples-per-sec"], "flows": ["--records-per-sec"]}, + ) + def test_one_process_per_sink_and_metric_spread_over_workers(self): commands = self.start(running_count=2) feeders = [(n, c) for n, c in commands if "--sink" in c] diff --git a/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py b/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py index 1b31bc9d..72f795ad 100644 --- a/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py +++ b/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py @@ -152,6 +152,12 @@ def test_structured_queries_from_sql_file(self): group["queries"], ["SELECT dst_subnet, uniq(src_ip) FROM flows GROUP BY dst_subnet"], ) + # The client refuses to render {{ end_time }} templates without a + # window; the SQL carries its own, so 0. PromQL groups have none. + self.assertEqual(group["time_window_seconds"], 0) + self.assertNotIn( + "time_window_seconds", config["experiment_params"]["query_groups"][0] + ) self.assertFalse(any("structured" in n for n in notes)) # The records metric goes to ASAPQuery only: fed, but no baseline # mode and not under `metrics`. @@ -168,6 +174,17 @@ def test_structured_queries_from_sql_file(self): "no baseline mode: records metrics ['flows'] go to ASAPQuery only", notes ) + def test_records_rate_only_with_records_metrics(self): + sql = os.path.join(self.tmp.name, "multigroup.sql") + with open(sql, "w") as f: + f.write("SELECT 1;\n") + config, _ = self.build(sql_queries_file=sql, records_per_sec=1000) + feeder = config["experiment_params"]["exporters"]["exporter_list"] + self.assertEqual(feeder["paced_feeder"]["records_per_sec"], 1000) + config, _ = self.build(records_per_sec=1000) + feeder = config["experiment_params"]["exporters"]["exporter_list"] + self.assertNotIn("records_per_sec", feeder["paced_feeder"]) + def test_sql_file_must_match_the_structured_queries(self): sql = os.path.join(self.tmp.name, "multigroup.sql") with open(sql, "w") as f: From 4a7c2054d25644d671789fdf1ea04b5953387a51 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 13 Oct 2026 14:03:48 +0000 Subject: [PATCH 7/9] revert(asap-tools): drop Prometheus' out-of-order window for the paced feeder The feeder now keeps each series' samples of a tick in one ordered request, and Prometheus 2.53's remote-write receiver accepted a full feed (48M samples) with no out-of-order window and no rejections. This reverts f7e5a66; Prometheus keeps --web.enable-remote-write-receiver. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AnJ7LocWxguh3q4QWkCYae --- .../CONFIG_PARAMETERS_REFERENCE.md | 4 +- asap-tools/experiments/constants.py | 4 -- .../experiments/generate_prometheus_config.py | 10 +--- .../experiments/tests/test_paced_feeder.py | 52 +------------------ 4 files changed, 3 insertions(+), 67 deletions(-) diff --git a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md index dbbef000..f1a61f3c 100644 --- a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md +++ b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md @@ -465,9 +465,7 @@ one process per (sink, metric). It pushes over remote write instead of being scraped: in `sketchdb` mode to the query engine's ingest (`streaming.remote_write`, which must have `parallelism: 1`), plus Prometheus when the mode sets `query_prometheus_too`; in every other mode to Prometheus, -which then starts with `--web.enable-remote-write-receiver` and a 5 s -`storage.tsdb.out_of_order_time_window` (a tick's requests are sent -concurrently, so samples can arrive out of order within a second). All processes +which then starts with `--web.enable-remote-write-receiver`. All processes share the seed and table, so every system gets the same data. Its metrics are listed under `metrics` with `exporter: paced_feeder`; Prometheus does not forward them to the query engine. Generate a whole config from a sketch-bench diff --git a/asap-tools/experiments/constants.py b/asap-tools/experiments/constants.py index 151999af..2faf6967 100644 --- a/asap-tools/experiments/constants.py +++ b/asap-tools/experiments/constants.py @@ -44,10 +44,6 @@ PROMETHEUS_REMOTE_WRITE_ENDPOINT = "/api/v1/write" # Lets Prometheus accept samples pushed to PROMETHEUS_REMOTE_WRITE_ENDPOINT. PROMETHEUS_REMOTE_WRITE_RECEIVER_FLAG = "--web.enable-remote-write-receiver" -# The paced feeder sends a tick's requests concurrently, so a series' samples -# can arrive out of order within that second; Prometheus rejects them without -# an out-of-order window. -PROMETHEUS_OUT_OF_ORDER_TIME_WINDOW = "5s" HTTP_READINESS_CURL_TIMEOUT_SECONDS = 5 QUERY_ENGINE_RUNTIME_INFO_ENDPOINT = "/api/v1/status/runtimeinfo" diff --git a/asap-tools/experiments/generate_prometheus_config.py b/asap-tools/experiments/generate_prometheus_config.py index bcd3beee..a988ee72 100644 --- a/asap-tools/experiments/generate_prometheus_config.py +++ b/asap-tools/experiments/generate_prometheus_config.py @@ -4,9 +4,7 @@ import argparse from omegaconf import DictConfig -import constants import experiment_utils -from experiment_utils.services.paced_feeder import PACED_FEEDER_EXPORTER from utils import get_node_range @@ -160,13 +158,6 @@ def main(args, experiment_config=None): if exporter_config is None: raise ValueError("Invalid exporter config: {}".format(rejection_reason)) - if PACED_FEEDER_EXPORTER in exporter_config["exporter_list"]: - config["storage"] = { - "tsdb": { - "out_of_order_time_window": constants.PROMETHEUS_OUT_OF_ORDER_TIME_WINDOW - } - } - # add section for "scrape_configs" config["scrape_configs"] = [] # add target for self-monitoring prometheus @@ -446,6 +437,7 @@ def main(args, experiment_config=None): config["remote_write"].append(remote_write_config) # Write prometheus config to output_dir + import constants output_file = os.path.join(args.output_dir, constants.PROMETHEUS_CONFIG_FILE) os.makedirs(args.output_dir, exist_ok=True) diff --git a/asap-tools/experiments/tests/test_paced_feeder.py b/asap-tools/experiments/tests/test_paced_feeder.py index 1484a80b..44a27f14 100644 --- a/asap-tools/experiments/tests/test_paced_feeder.py +++ b/asap-tools/experiments/tests/test_paced_feeder.py @@ -7,17 +7,10 @@ import unittest from unittest import mock -import yaml - from omegaconf import OmegaConf import constants -import generate_prometheus_config -from experiment_utils.core import ( - GeneratePrometheusArgs, - call_generate_prometheus_config, - read_exporter_config, -) +from experiment_utils.core import call_generate_prometheus_config, read_exporter_config from experiment_utils.services import create_prometheus_service from experiment_utils.services.docker_prometheus import DockerPrometheusService from experiment_utils.services.paced_feeder import ( @@ -474,48 +467,5 @@ def test_scraped_metrics_are_still_forwarded(self): self.assertEqual(args.remote_write_metric_names, ["node_cpu"]) -class PrometheusOutOfOrderWindowTest(unittest.TestCase): - """The feeder sends a tick's requests concurrently, so Prometheus' receiver - sees a series' samples out of order within a second and, without an - out-of-order window, rejects them (400 "out of order sample"). Bare-metal - and containerized Prometheus both read this generated prometheus.yml.""" - - def storage(self, exporter_list): - with tempfile.TemporaryDirectory() as out: - prometheus_config = OmegaConf.create( - { - "scrape_interval": "1s", - "evaluation_interval": "10s", - "recording_rules": {"interval": "5s"}, - } - ) - args = GeneratePrometheusArgs( - 1, out, out, prometheus_config, "10.10.1.1", "10.10.1", 0 - ) - generate_prometheus_config.main( - args, - { - "exporters": { - "only_start_if_queries_exist": False, - "exporter_list": exporter_list, - }, - "query_groups": [], - "metrics": [], - }, - ) - with open(os.path.join(out, constants.PROMETHEUS_CONFIG_FILE)) as f: - return yaml.safe_load(f).get("storage") - - def test_window_set_when_the_feeder_targets_prometheus(self): - feeder = {"workload_table": "t", "metrics": [], "seed": 1, "duration_secs": 1} - self.assertEqual( - self.storage({"paced_feeder": feeder}), - {"tsdb": {"out_of_order_time_window": "5s"}}, - ) - - def test_no_window_without_the_feeder(self): - self.assertIsNone(self.storage({})) - - if __name__ == "__main__": unittest.main() From cced5a4f19437fac86682ae5f2ec6874e6944ab8 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 09:57:36 -0400 Subject: [PATCH 8/9] fix(asap-tools): give every e2e mode the same data windows Each mode's feeder now stamps its data from the last multiple of the aligned query intervals (sending it in real time from a shared send time), and aligned query groups evaluate at that origin + k*T, sent once the data has arrived. Every mode then answers over the same samples, evaluation times stay on ASAPQuery's windows, and the first window is never empty. Also from review: - refuse forward_unsupported_queries when sketchdb mode gives Prometheus no data (the generator sets it false), and auto_discover_labels - take the fed metrics from the feeder block - launch feeders with one SSH per node and fail if that overruns the lead - stop feeders at exit after a failed run - resolve a relative workload_table; check parallelism only for sketchdb Co-Authored-By: Claude Opus 5.5 --- .../CONFIG_PARAMETERS_REFERENCE.md | 8 +- asap-tools/experiments/experiment_run_e2e.py | 31 ++- .../experiment_utils/services/__init__.py | 4 + .../experiment_utils/services/paced_feeder.py | 110 ++++++++-- .../sketch_bench_e2e_configs.py | 5 +- .../experiments/tests/test_paced_feeder.py | 194 ++++++++++++++++-- .../tests/test_sketch_bench_e2e_configs.py | 27 ++- .../prometheus-client/classes/query_time.py | 52 +++-- .../main_prometheus_client.py | 23 ++- .../tests/test_query_time.py | 48 +++-- 10 files changed, 411 insertions(+), 91 deletions(-) diff --git a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md index f1a61f3c..75addbc7 100644 --- a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md +++ b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md @@ -543,9 +543,13 @@ table with `sketch_bench_e2e/sketch_bench_e2e_configs.py`. - **Usage**: Ensures data is available for lookback queries #### `experiment_params.query_groups[].client_options.align_to_interval` (boolean, optional) -- **Description**: Evaluate repetition k at `t_k = floor((start - query_time_offset) / T) * T + k * T`, with `T = repetition_delay_ms` (whole seconds), sending it at wall-clock `t_k + query_time_offset` +- **Description**: Evaluate repetition k at `t_k = data_start_time + (max(1, ceil(starting_delay / T)) + k) * T`, with `T = repetition_delay_ms` (whole seconds), sending it at wall-clock `t_k + data_lag_secs + query_time_offset`. `starting_delay` counts in event time from `data_start_time`, not as a wall-clock sleep - **Default**: `false` -- **Usage**: Every evaluation time is a multiple of `T`, so each system answers the same windows; `query_time_offset` is how long the data of the window ending at `t_k` gets to arrive. Overrides the query engine's per-aggregation alignment in sketchdb mode +- **Usage**: Every evaluation time is a multiple of `T`, and repetition k covers the same samples in every mode however late the mode started; `query_time_offset` is how long the data of the window ending at `t_k` gets to arrive. Overrides the query engine's per-aggregation alignment in sketchdb mode. Requires `data_start_time` and `data_lag_secs` + +#### `experiment_params.query_groups[].client_options.data_start_time` / `data_lag_secs` (int / float) +- **Description**: Event time (epoch seconds) the paced feeder's data starts at, and how many seconds it runs behind the wall clock +- **Usage**: Not set by hand: `experiment_run_e2e.py` writes them into each mode's client config after starting the paced feeder, which backdates its data to the last multiple of the aligned groups' intervals #### `experiment_params.query_groups[].client_options.starting_delay` (int, required) - **Description**: Initial delay in seconds before starting query execution diff --git a/asap-tools/experiments/experiment_run_e2e.py b/asap-tools/experiments/experiment_run_e2e.py index d5115548..8334a0a9 100644 --- a/asap-tools/experiments/experiment_run_e2e.py +++ b/asap-tools/experiments/experiment_run_e2e.py @@ -1,8 +1,10 @@ import os +import atexit import json import time import hydra +import yaml from omegaconf import DictConfig, OmegaConf import constants @@ -27,9 +29,11 @@ SystemExportersService, PACED_FEEDER_EXPORTER, PacedFeederService, + aligned_interval_lcm, check_paced_feeder_config, paced_feeder_min_duration_secs, paced_feeder_sinks, + set_data_start, ) from experiment_utils.services.misc import DiscoveryBackend @@ -201,6 +205,10 @@ def main(cfg: DictConfig): paced_feeder_config = exporter_config["exporter_list"].get(PACED_FEEDER_EXPORTER) if paced_feeder_config is not None: paced_feeder_config = OmegaConf.to_container(paced_feeder_config, resolve=True) + # The feeder runs from its own directory. + paced_feeder_config["workload_table"] = os.path.abspath( + paced_feeder_config["workload_table"] + ) check_paced_feeder_config( paced_feeder_config, modes=list(modes_by_name.values()), @@ -209,8 +217,12 @@ def main(cfg: DictConfig): 0 if skip_querying else args.steady_state_wait, minimum_experiment_running_time, ), + forward_unsupported_queries=args.forward_unsupported_queries, + auto_discover_labels=args.controller_auto_discover_labels, ) paced_feeder_service = PacedFeederService(provider, args) + # Otherwise a failed run leaves feeders pushing into the next run's sinks. + atexit.register(paced_feeder_service.stop) for experiment_mode in experiment_modes: print(f"Running experiment mode: {experiment_mode}") @@ -517,11 +529,28 @@ def main(cfg: DictConfig): ) if prometheus_url in sinks.values(): prometheus_service.wait_until_ready() - paced_feeder_service.start( + local_client_config = os.path.join( + local_experiment_root_dir, + "controller_client_configs", + f"{experiment_mode}.yaml", + ) + with open(local_client_config) as f: + client_config = yaml.safe_load(f) + start_ms, send_at_ms = paced_feeder_service.start( config=paced_feeder_config, sinks=sinks, experiment_output_dir=experiment_output_dir, local_experiment_dir=local_experiment_dir, + align_secs=aligned_interval_lcm(client_config), + ) + set_data_start(client_config, start_ms, send_at_ms) + with open(local_client_config, "w") as f: + yaml.safe_dump(client_config, f, sort_keys=False) + sync.rsync_controller_client_configs( + provider, + experiment_root_output_dir, + local_experiment_root_dir, + node_offset=args.node_offset, ) # Start Prometheus throughput monitoring if enabled diff --git a/asap-tools/experiments/experiment_utils/services/__init__.py b/asap-tools/experiments/experiment_utils/services/__init__.py index 32cd0d53..fbb16d94 100644 --- a/asap-tools/experiments/experiment_utils/services/__init__.py +++ b/asap-tools/experiments/experiment_utils/services/__init__.py @@ -37,9 +37,11 @@ from .paced_feeder import ( PACED_FEEDER_EXPORTER, PacedFeederService, + aligned_interval_lcm, check_paced_feeder_config, paced_feeder_min_duration_secs, paced_feeder_sinks, + set_data_start, ) from .clickhouse_service import ClickHouseService, ClickHouseDataLoaderService @@ -164,7 +166,9 @@ def create_prometheus_service(cfg, provider, num_nodes: int, node_offset: int): "ClickHouseDataLoaderService", "PACED_FEEDER_EXPORTER", "PacedFeederService", + "aligned_interval_lcm", "check_paced_feeder_config", "paced_feeder_min_duration_secs", "paced_feeder_sinks", + "set_data_start", ] diff --git a/asap-tools/experiments/experiment_utils/services/paced_feeder.py b/asap-tools/experiments/experiment_utils/services/paced_feeder.py index 94095324..7953946f 100644 --- a/asap-tools/experiments/experiment_utils/services/paced_feeder.py +++ b/asap-tools/experiments/experiment_utils/services/paced_feeder.py @@ -7,6 +7,7 @@ the same seed and table, so each system receives identical data. """ +import functools import json import math import os @@ -30,8 +31,8 @@ PACED_FEEDER_BINARY = "./target/release/paced_feeder" # The brackets keep pgrep/pkill from matching the shell that runs them. PACED_FEEDER_PROCESS_PATTERN = "[t]arget/release/paced_feeder" -# Feeders launched one after another over SSH all start at this much later -# event time, so the sinks of one mode get the same timestamps. +# Feeders are launched over SSH, one node after another, and all send their +# first tick this long after the launches begin. PACED_FEEDER_START_LEAD_SECS = 5 # A feeder stops after its first tick if every request fails (e.g. a wrong # URL), so check the processes one tick after the shared start. @@ -42,12 +43,42 @@ def paced_feeder_metric_names(experiment_params) -> List[str]: - """Metrics the paced feeder sends, per the experiment's `metrics` list.""" - return [ - m["metric"] - for m in experiment_params.get("metrics", []) - if m["exporter"] == PACED_FEEDER_EXPORTER + """Metrics the paced feeder sends, per its own config block.""" + exporter_list = experiment_params.get("exporters", {}).get("exporter_list", {}) + feeder = exporter_list.get(PACED_FEEDER_EXPORTER) + return list(feeder["metrics"]) if feeder is not None else [] + + +def backdated_start_ms(send_at_ms: int, align_secs: int) -> int: + """The last multiple of `align_secs` at or before `send_at_ms`: the feeder + stamps its data from there, so every mode's data starts on the same + boundary of each aligned query group's interval.""" + align_ms = align_secs * 1000 + return send_at_ms // align_ms * align_ms + + +def aligned_interval_lcm(client_config: Dict[str, Any]) -> int: + """LCM, in seconds, of the repeat intervals of the client config's + `align_to_interval` groups; 1 if there are none.""" + intervals = [ + g["repetition_delay_ms"] // 1000 + for g in client_config["query_groups"] + if g.get("client_options", {}).get("align_to_interval") ] + # math.lcm needs Python 3.9. + return functools.reduce(lambda a, b: a * b // math.gcd(a, b), intervals, 1) + + +def set_data_start( + client_config: Dict[str, Any], start_ms: int, send_at_ms: int +) -> None: + """Tell the client's aligned groups when the data starts in event time and + how far it runs behind the wall clock (see the client's query_time.py).""" + for group in client_config["query_groups"]: + options = group.get("client_options", {}) + if options.get("align_to_interval"): + options["data_start_time"] = start_ms // 1000 + options["data_lag_secs"] = (send_at_ms - start_ms) / 1000 def feeds_prometheus(mode: Dict[str, Any]) -> bool: @@ -62,8 +93,8 @@ def paced_feeder_min_duration_secs( ) -> float: """Seconds of data that last until the final query: queries end at most startup check + steady_state_wait + the query groups' running time after - launch, and the data starts PACED_FEEDER_START_LEAD_SECS after launch (not - subtracted, for slack).""" + launch. Aligned groups need less (their last evaluation is at most the + starting delay + running time into the data), so this covers both.""" return PACED_FEEDER_STARTUP_CHECK_SECS + steady_state_wait + query_running_secs @@ -79,11 +110,28 @@ def check_paced_feeder_config( modes: List[Dict[str, Any]], parallelism: int, min_feed_secs: float, + forward_unsupported_queries: bool, + auto_discover_labels: bool, ) -> None: """Fail before the experiment starts on a config the feeder would reject, - one sending records to Prometheus, or one that stops sending before the - queries end.""" + one sending records to Prometheus, one that stops sending before the + queries end, or one whose queries would read a Prometheus without the + data.""" prometheus_modes = [m["mode"] for m in modes if feeds_prometheus(m)] + sketchdb_modes = [ + m for m in modes if m["mode"] == constants.SKETCHDB_EXPERIMENT_NAME + ] + if forward_unsupported_queries and not all(map(feeds_prometheus, sketchdb_modes)): + raise ValueError( + "sketchdb mode without query_prometheus_too gives Prometheus no " + "paced_feeder data, so queries ASAPQuery forwards there would come " + "back empty; set streaming.forward_unsupported_queries=false" + ) + if auto_discover_labels: + raise ValueError( + "controller.auto_discover_labels reads labels from Prometheus before " + "the paced_feeder starts; list the labels under experiment_params.metrics" + ) table_metrics = table_metric_types(config["workload_table"]) for metric in config["metrics"]: if metric not in table_metrics: @@ -103,7 +151,7 @@ def check_paced_feeder_config( raise ValueError( f"paced_feeder metric {metric!r} has unknown type {metric_type!r}" ) - if parallelism != 1: + if sketchdb_modes and parallelism != 1: raise ValueError( "paced_feeder sends to one ingest port; set streaming.parallelism=1" ) @@ -139,13 +187,15 @@ def paced_feeder_command( metric: str, metric_type: str, start_ms: int, + send_at_ms: int, stats_out: str, ) -> str: cmd = ( f"{PACED_FEEDER_BINARY} --sink remote-write --url {url}" f" --workload-table {workload_table} --metric {metric}" f" --seed {config['seed']} --duration-secs {config['duration_secs']}" - f" --start-ms {start_ms} --stats-out {stats_out}" + f" --start-ms {start_ms} --send-at-ms {send_at_ms}" + f" --stats-out {stats_out}" ) # The feeder refuses the other kind's rate flag. if metric_type == RECORDS_METRIC_TYPE: @@ -179,8 +229,11 @@ def start( sinks: Dict[str, str], experiment_output_dir: str, local_experiment_dir: str, + align_secs: int = 1, **kwargs, - ) -> None: + ) -> Tuple[int, int]: + """Launch the feeders; returns the data's event-time start and the + wall-clock time its first tick is sent, both in epoch ms.""" output_dir = os.path.join(experiment_output_dir, "paced_feeder_output") metric_types = table_metric_types(config["workload_table"]) jobs = [(sink, metric) for sink in sinks for metric in config["metrics"]] @@ -203,9 +256,10 @@ def start( self.provider, config["workload_table"], output_dir, node_idx ) - # One shared event-time origin for every process of this mode, taken - # after the SSH prep above so the lead covers only the launches. - start_ms = (math.ceil(time.time()) + PACED_FEEDER_START_LEAD_SECS) * 1000 + # One shared send time for every process of this mode, taken after the + # SSH prep above so the lead covers only the launches. + send_at_ms = (math.ceil(time.time()) + PACED_FEEDER_START_LEAD_SECS) * 1000 + start_ms = backdated_start_ms(send_at_ms, align_secs) node_cmds: List[Tuple[int, str]] = [] for node_idx, (sink, metric) in zip(job_nodes, jobs): name = f"{sink}_{metric}" @@ -216,10 +270,14 @@ def start( metric, metric_types[metric], start_ms, + send_at_ms, os.path.join(output_dir, f"{name}_stats.json"), ) node_cmds.append( - (node_idx, f"{cmd} > {os.path.join(output_dir, name)}.out 2>&1 &") + ( + node_idx, + f"nohup {cmd} > {os.path.join(output_dir, name)}.out 2>&1 &", + ) ) local_config_dir = os.path.join(local_experiment_dir, "paced_feeder_config") @@ -227,15 +285,24 @@ def start( with open(os.path.join(local_config_dir, "cmds.sh"), "w") as f: f.write("\n".join(cmd for _, cmd in node_cmds) + "\n") - for node_idx, cmd in node_cmds: + # One SSH per node, so the launches fit in the lead. + for node_idx in used_nodes: self.provider.execute_command( node_idx=node_idx, - cmd=cmd, + cmd=" ".join(cmd for n, cmd in node_cmds if n == node_idx), cmd_dir=self.cmd_dir, - nohup=True, + nohup=False, popen=False, ignore_errors=False, ) + if time.time() * 1000 > send_at_ms: + # A feeder launched after its send time starts late, so its data + # would trail the other sinks'. + self.stop() + raise RuntimeError( + f"paced_feeder: launching on {len(used_nodes)} nodes took longer " + f"than the {PACED_FEEDER_START_LEAD_SECS} s lead" + ) time.sleep(PACED_FEEDER_STARTUP_CHECK_SECS) for node_idx in used_nodes: @@ -254,6 +321,7 @@ def start( f"paced_feeder: expected {expected} processes running on " f"node {node_idx}; see the .out files in {output_dir}" ) + return start_ms, send_at_ms def stop(self, **kwargs) -> None: self.provider.execute_command_parallel( diff --git a/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py b/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py index dd7bee6a..9bfec0cf 100644 --- a/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py +++ b/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py @@ -18,6 +18,8 @@ Prometheus' forwarding, and records are never scraped). - `controller.prebuilt_output_dir`, when given, so ASAPQuery runs those planner configs (e.g. from `asap-planner --planner saved`). +- `streaming.forward_unsupported_queries=false`: Prometheus gets no data in + sketchdb mode, so a forwarded query would come back empty. Usage (from asap-tools/experiments): python sketch_bench_e2e/sketch_bench_e2e_configs.py \\ @@ -199,7 +201,8 @@ def build_config( }, "query_groups": query_groups, "metrics": metrics, - } + }, + "streaming": {"forward_unsupported_queries": False}, } if prebuilt_output_dir is not None: prebuilt_output_dir = os.path.abspath(prebuilt_output_dir) diff --git a/asap-tools/experiments/tests/test_paced_feeder.py b/asap-tools/experiments/tests/test_paced_feeder.py index 44a27f14..f3409afb 100644 --- a/asap-tools/experiments/tests/test_paced_feeder.py +++ b/asap-tools/experiments/tests/test_paced_feeder.py @@ -13,14 +13,17 @@ from experiment_utils.core import call_generate_prometheus_config, read_exporter_config from experiment_utils.services import create_prometheus_service from experiment_utils.services.docker_prometheus import DockerPrometheusService +from experiment_utils.services import paced_feeder from experiment_utils.services.paced_feeder import ( PACED_FEEDER_START_LEAD_SECS, PACED_FEEDER_STARTUP_CHECK_SECS, PacedFeederService, - check_paced_feeder_config, + aligned_interval_lcm, + backdated_start_ms, paced_feeder_min_duration_secs, paced_feeder_command, paced_feeder_sinks, + set_data_start, ) from experiment_utils.services.prometheus import PrometheusService @@ -67,6 +70,24 @@ def write_table(directory, queries=()): return path +def check_paced_feeder_config( + config, + modes, + parallelism, + min_feed_secs, + forward_unsupported_queries=False, + auto_discover_labels=False, +): + paced_feeder.check_paced_feeder_config( + config, + modes, + parallelism, + min_feed_secs, + forward_unsupported_queries=forward_unsupported_queries, + auto_discover_labels=auto_discover_labels, + ) + + def feeder_config(table, **overrides): return { "workload_table": table, @@ -85,6 +106,7 @@ def test_command_line(self): QE_URL, "data", "gauge", + 1700000000000, 1700000005000, "/out/query_engine_data_stats.json", ) @@ -92,7 +114,8 @@ def test_command_line(self): cmd, "./target/release/paced_feeder --sink remote-write" f" --url {QE_URL} --workload-table /out/table.json --metric data" - " --seed 42 --duration-secs 600 --start-ms 1700000005000" + " --seed 42 --duration-secs 600 --start-ms 1700000000000" + " --send-at-ms 1700000005000" " --stats-out /out/query_engine_data_stats.json", ) @@ -104,6 +127,7 @@ def test_optional_flags_are_appended(self): "data", "gauge", 0, + 0, "/s", ) self.assertTrue(cmd.endswith(" --samples-per-sec 20 --concurrency 4")) @@ -116,7 +140,7 @@ def test_each_metric_gets_its_own_kind_of_rate(self): def rate_flags(metric, metric_type): cmd = paced_feeder_command( - config, "/t", QE_URL, metric, metric_type, 0, "/s" + config, "/t", QE_URL, metric, metric_type, 0, 0, "/s" ) return [f for f in cmd.split() if f.endswith("-per-sec")] @@ -193,6 +217,39 @@ def test_sharded_ingest(self): with self.assertRaisesRegex(ValueError, "parallelism=1"): check_paced_feeder_config(feeder_config(self.table), BOTH_MODES, 2, 600) + def test_sharded_ingest_is_fine_without_sketchdb(self): + check_paced_feeder_config(feeder_config(self.table), [BASELINE], 2, 600) + + def test_forwarding_to_a_prometheus_without_data_refused(self): + # In sketchdb mode Prometheus gets no fed data, so a query ASAPQuery + # can't answer would be forwarded there and come back empty. + with self.assertRaisesRegex(ValueError, "forward_unsupported_queries"): + check_paced_feeder_config( + feeder_config(self.table), + BOTH_MODES, + 1, + 600, + forward_unsupported_queries=True, + ) + check_paced_feeder_config( + feeder_config(self.table), + [{**SKETCHDB, "query_prometheus_too": True}, BASELINE], + 1, + 600, + forward_unsupported_queries=True, + ) + + def test_label_discovery_refused(self): + # It reads Prometheus before the feeder has sent anything. + with self.assertRaisesRegex(ValueError, "auto_discover_labels"): + check_paced_feeder_config( + feeder_config(self.table), + BOTH_MODES, + 1, + 600, + auto_discover_labels=True, + ) + def test_feeder_ending_before_the_queries(self): with self.assertRaisesRegex(ValueError, "at least 601"): check_paced_feeder_config(feeder_config(self.table), BOTH_MODES, 1, 601) @@ -223,14 +280,60 @@ def test_missing_keys_rejected_with_the_other_exporters(self): self.assertIn("duration_secs", reason) +def client_config(*intervals_secs, aligned=True): + return { + "query_groups": [ + { + "id": i, + "repetition_delay_ms": t * 1000, + "client_options": {"repetitions": 2, "align_to_interval": aligned}, + } + for i, t in enumerate(intervals_secs) + ] + } + + +class DataStartTest(unittest.TestCase): + def test_backdates_to_the_last_boundary(self): + # Sent from 10:02:23, stamped from 10:00:00. + send_at = 1791885743000 + self.assertEqual(backdated_start_ms(send_at, 300), 1791885600000) + self.assertEqual(backdated_start_ms(1791885600000, 300), 1791885600000) + self.assertEqual(backdated_start_ms(send_at, 1), send_at) + + def test_origin_fits_every_aligned_interval(self): + self.assertEqual(aligned_interval_lcm(client_config(1, 60, 300)), 300) + self.assertEqual(aligned_interval_lcm(client_config(60, 90)), 180) + self.assertEqual(aligned_interval_lcm(client_config(60, aligned=False)), 1) + + def test_aligned_groups_learn_the_data_start(self): + config = client_config(60) + config["query_groups"].append( + {"id": 9, "repetition_delay_ms": 1000, "client_options": {}} + ) + set_data_start(config, 1791885600000, 1791885743000) + self.assertEqual( + config["query_groups"][0]["client_options"], + { + "repetitions": 2, + "align_to_interval": True, + "data_start_time": 1791885600, + "data_lag_secs": 143.0, + }, + ) + self.assertEqual(config["query_groups"][1]["client_options"], {}) + + # Fake seconds each SSH prep step (binary check, table copy) takes. PREP_SECS = 10 class RecordingProvider: - def __init__(self, running_count): + def __init__(self, running_count, launch_secs=0): self.commands = [] + self.launches = 0 self.running_count = running_count + self.launch_secs = launch_secs self.clock = 1700000000.0 def get_home_dir(self): @@ -239,8 +342,19 @@ def get_home_dir(self): def is_remote(self): return True + def execute_command_parallel(self, node_idxs, cmd, **kwargs): + self.commands += [(n, cmd) for n in node_idxs] + def execute_command(self, node_idx, cmd, **kwargs): - self.commands.append((node_idx, cmd)) + if "--sink" in cmd: + # One SSH launches every feeder of a node. + self.launches += 1 + self.clock += self.launch_secs + self.commands += [ + (node_idx, f"nohup {c.strip()}") for c in cmd.split("nohup ") if c + ] + else: + self.commands.append((node_idx, cmd)) if cmd.startswith("test -x"): self.clock += PREP_SECS returncode = 0 @@ -258,8 +372,15 @@ def get_coordinator_node(self): class PacedFeederServiceStartTest(unittest.TestCase): - def start(self, running_count, sinks=None, **config_overrides): - provider = RecordingProvider(running_count) + def start( + self, + running_count, + sinks=None, + align_secs=1, + launch_secs=0, + **config_overrides, + ): + provider = RecordingProvider(running_count, launch_secs) self.provider = provider service = PacedFeederService(provider, Args()) @@ -276,26 +397,48 @@ def rsync(*args): "experiment_utils.services.paced_feeder.sync.rsync_dataset_file", side_effect=rsync, ): - service.start( + self.started = service.start( config=feeder_config(write_table(local_dir), **config_overrides), sinks=sinks or {"query_engine": QE_URL, "prometheus": PROM_URL}, experiment_output_dir="/exp", local_experiment_dir=local_dir, + align_secs=align_secs, ) return provider.commands - def test_start_time_leads_the_launches_not_the_prep(self): - # The SSH prep (2 nodes x binary check + table copy) takes 40 fake - # seconds; the shared start must still be the lead after it. - commands = self.start(running_count=2) - start_secs = { - int(c.split(" --start-ms ")[1].split()[0]) / 1000 + @staticmethod + def flag_values(commands, flag): + return { + int(c.split(f" {flag} ")[1].split()[0]) for _, c in commands if "--sink" in c } - self.assertEqual( - start_secs, {1700000000 + 4 * PREP_SECS + PACED_FEEDER_START_LEAD_SECS} - ) + + def test_send_time_leads_the_launches_not_the_prep(self): + # The SSH prep (2 nodes x binary check + table copy) takes 40 fake + # seconds; the shared send time must still be the lead after it. + commands = self.start(running_count=2) + send_at_ms = (1700000000 + 4 * PREP_SECS + PACED_FEEDER_START_LEAD_SECS) * 1000 + self.assertEqual(self.flag_values(commands, "--send-at-ms"), {send_at_ms}) + self.assertEqual(self.flag_values(commands, "--start-ms"), {send_at_ms}) + self.assertEqual(self.started, (send_at_ms, send_at_ms)) + + def test_data_is_backdated_to_the_aligned_boundary(self): + # Sent from 1700000045 s, stamped from the last multiple of 300 s. + commands = self.start(running_count=2, align_secs=300) + self.assertEqual(self.flag_values(commands, "--start-ms"), {1699999800000}) + self.assertEqual(self.flag_values(commands, "--send-at-ms"), {1700000045000}) + self.assertEqual(self.started, (1699999800000, 1700000045000)) + + def test_one_launch_per_node(self): + self.start(running_count=2) + self.assertEqual(self.provider.launches, 2) + + def test_launch_slower_than_the_lead_fails(self): + # A feeder launched after its send time would trail the other sinks. + with self.assertRaisesRegex(RuntimeError, "longer than the 5 s lead"): + self.start(running_count=2, launch_secs=3) + self.assertTrue(any("pkill" in c for _, c in self.provider.commands)) def test_records_and_gauge_processes_get_their_rate_flags(self): commands = self.start( @@ -331,8 +474,8 @@ def test_one_process_per_sink_and_metric_spread_over_workers(self): ], [ (1, QE_URL, "data"), - (2, QE_URL, "data_total"), (1, PROM_URL, "data"), + (2, QE_URL, "data_total"), (2, PROM_URL, "data_total"), ], ) @@ -417,7 +560,7 @@ class PrometheusForwardingTest(unittest.TestCase): """In sketchdb mode Prometheus must not forward fed metrics to the query engine, which the feeder already writes to.""" - def remote_write_args(self, metrics): + def remote_write_args(self, metrics, fed=("data",)): cfg = OmegaConf.create( { "streaming": { @@ -431,7 +574,7 @@ def remote_write_args(self, metrics): "experiment_params": { "exporters": { "only_start_if_queries_exist": False, - "exporter_list": {}, + "exporter_list": {"paced_feeder": {"metrics": list(fed)}}, }, "query_groups": [], "metrics": metrics, @@ -466,6 +609,17 @@ def test_scraped_metrics_are_still_forwarded(self): self.assertEqual(args.remote_write_url, QE_URL) self.assertEqual(args.remote_write_metric_names, ["node_cpu"]) + def test_fed_metrics_come_from_the_feeder_block(self): + # The feeder sends what its own block lists; a metric not tagged + # paced_feeder in `metrics` would otherwise reach ASAPQuery twice. + args = self.remote_write_args( + [ + {"metric": "data", "labels": ["job"], "exporter": "node_exporter"}, + {"metric": "node_cpu", "labels": ["job"], "exporter": "node_exporter"}, + ] + ) + self.assertEqual(args.remote_write_metric_names, ["node_cpu"]) + if __name__ == "__main__": unittest.main() diff --git a/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py b/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py index 72f795ad..15dafe76 100644 --- a/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py +++ b/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py @@ -6,12 +6,17 @@ import tempfile import unittest +import yaml from hydra import compose, initialize_config_dir from omegaconf import OmegaConf from experiment_utils import config as experiment_config from experiment_utils.core import read_exporter_config -from experiment_utils.services.paced_feeder import check_paced_feeder_config +from experiment_utils.services.paced_feeder import ( + aligned_interval_lcm, + check_paced_feeder_config, + set_data_start, +) from sketch_bench_e2e import sketch_bench_e2e_configs as sbe from test_paced_feeder import E2E_METRICS @@ -230,6 +235,9 @@ def compose_and_validate(self, config): parallelism=cfg.streaming.parallelism, min_feed_secs=cfg.flow.steady_state_wait + experiment_config.get_minimum_experiment_running_time(params), + # The generated config overrides config.yaml's `true`. + forward_unsupported_queries=cfg.streaming.forward_unsupported_queries, + auto_discover_labels=cfg.controller.auto_discover_labels, ) modes, _ = experiment_config.generate_controller_client_configs( params, self.tmp.name @@ -241,6 +249,23 @@ def test_generated_config_composes_and_validates(self): self.compose_and_validate(self.build()[0]), ["sketchdb", "baseline"] ) + def test_runner_can_anchor_the_generated_client_config(self): + # What experiment_run_e2e.py does to the mode's client config once the + # feeder starts: every group must learn where the data starts. + self.compose_and_validate(self.build()[0]) + path = os.path.join(self.tmp.name, "controller_client_configs", "sketchdb.yaml") + with open(path) as f: + client_config = yaml.safe_load(f) + self.assertEqual(aligned_interval_lcm(client_config), 60) + set_data_start(client_config, 1791885600000, 1791885743000) + self.assertEqual( + { + (o["data_start_time"], o["data_lag_secs"]) + for o in (g["client_options"] for g in client_config["query_groups"]) + }, + {(1791885600, 143.0)}, + ) + def test_generated_config_with_records_validates(self): sql = os.path.join(self.tmp.name, "multigroup.sql") with open(sql, "w") as f: diff --git a/asap-tools/queriers/prometheus-client/classes/query_time.py b/asap-tools/queriers/prometheus-client/classes/query_time.py index af208c19..d42cf852 100644 --- a/asap-tools/queriers/prometheus-client/classes/query_time.py +++ b/asap-tools/queriers/prometheus-client/classes/query_time.py @@ -1,40 +1,54 @@ """Evaluation times aligned to a query group's repeat interval. -With `client_options.align_to_interval`, repetition k of a group with repeat -interval T (`repetition_delay_ms`) and settle offset O -(`client_options.query_time_offset`, seconds, default 0) evaluates at - - t_k = floor((start - O) / T) * T + k * T - -and is sent at wall-clock time t_k + O. Every evaluation time is a multiple of -T, so all systems answer the same windows (e.g. a 60 s query at whole minutes), -and O gives the data of the window ending at t_k time to arrive first. For -example, T = 60 s and O = 10 s, started at 12:00:35: repetitions evaluate at -12:00:00, 12:01:00, ... and are sent at 12:00:35 (late, so at once), -12:01:10, 12:02:10, ... +With `client_options.align_to_interval`, the data starts at event time S +(`client_options.data_start_time`, epoch seconds, a multiple of every aligned +group's interval) and runs L seconds behind the wall clock +(`client_options.data_lag_secs`). Repetition k of a group with repeat interval +T (`repetition_delay_ms`), starting delay D (`starting_delay`) and settle +offset O (`query_time_offset`, default 0) evaluates at + + t_k = S + (max(1, ceil(D / T)) + k) * T + +and is sent at wall-clock time t_k + L + O, once its data has arrived. Every +evaluation time is a multiple of T, matching ASAPQuery's epoch-aligned windows, +and repetition k covers the same samples in every run regardless of when the +run started. For example, T = 60 s, D = 120 s, O = 10 s, and data stamped from +10:00:00 but sent from 10:02:23 (L = 143 s): repetitions evaluate at 10:02:00, +10:03:00, ... and are sent at 10:04:33, 10:05:33, ... """ +import math + MS_PER_SEC = 1000 def first_aligned_eval_time( - now_secs: float, interval_ms: int, offset_secs: float + data_start_time: int, interval_ms: int, starting_delay_secs: float ) -> int: - """The latest multiple of the interval at least `offset_secs` before now.""" + """The first multiple of the interval at least `starting_delay_secs` (and + one full interval) after the data starts.""" if interval_ms <= 0 or interval_ms % MS_PER_SEC != 0: raise ValueError( f"align_to_interval needs a whole-second repetition_delay_ms, got {interval_ms}" ) - if offset_secs < 0: - raise ValueError(f"query_time_offset must be >= 0, got {offset_secs}") interval_secs = interval_ms // MS_PER_SEC - return int((now_secs - offset_secs) // interval_secs) * interval_secs + if data_start_time % interval_secs != 0: + raise ValueError( + f"data_start_time {data_start_time} is not a multiple of the " + f"{interval_secs} s interval" + ) + intervals = max(1, math.ceil(starting_delay_secs / interval_secs)) + return data_start_time + intervals * interval_secs def aligned_eval_time(first_eval_time: int, interval_ms: int, repetition: int) -> int: return first_eval_time + repetition * (interval_ms // MS_PER_SEC) -def secs_until_send(eval_time: int, offset_secs: float, now_secs: float) -> float: +def secs_until_send( + eval_time: int, data_lag_secs: float, offset_secs: float, now_secs: float +) -> float: """How long to wait before evaluating at `eval_time`; 0 when already late.""" - return max(0.0, eval_time + offset_secs - now_secs) + if offset_secs < 0: + raise ValueError(f"query_time_offset must be >= 0, got {offset_secs}") + return max(0.0, eval_time + data_lag_secs + offset_secs - now_secs) diff --git a/asap-tools/queriers/prometheus-client/main_prometheus_client.py b/asap-tools/queriers/prometheus-client/main_prometheus_client.py index de20ca7d..a50cecb1 100644 --- a/asap-tools/queriers/prometheus-client/main_prometheus_client.py +++ b/asap-tools/queriers/prometheus-client/main_prometheus_client.py @@ -293,7 +293,10 @@ def handle_query_group( streaming_serializer: Optional[Any] = None, ) -> ResultDict: logger.debug(f"Starting query group {query_group.id}") - if query_group.starting_delay: + # See classes/query_time.py. Aligned groups apply the starting delay in + # event time and ignore query_start_times. + align_to_interval = getattr(query_group, "align_to_interval", False) + if query_group.starting_delay and not align_to_interval: logger.debug( f"Waiting for {query_group.starting_delay} seconds before starting" ) @@ -321,18 +324,24 @@ def handle_query_group( for server_name in servers } - # See classes/query_time.py. Aligned groups ignore query_start_times. - align_to_interval = getattr(query_group, "align_to_interval", False) simulated_start_time = getattr(query_group, "simulated_start_time", None) if align_to_interval and simulated_start_time is not None: raise ValueError( - "align_to_interval follows the wall clock; it can't be combined " + "align_to_interval follows data_start_time; it can't be combined " "with simulated_start_time" ) if align_to_interval: + data_start_time = getattr(query_group, "data_start_time", None) + data_lag_secs = getattr(query_group, "data_lag_secs", None) + if data_start_time is None or data_lag_secs is None: + raise ValueError( + "align_to_interval needs client_options.data_start_time and " + "data_lag_secs (set by the paced feeder runner)" + ) + lag_secs = float(data_lag_secs) offset_secs = getattr(query_group, "query_time_offset", 0) first_eval_time = first_aligned_eval_time( - time.time(), query_group.repetition_delay_ms, offset_secs + data_start_time, query_group.repetition_delay_ms, query_group.starting_delay ) for repetition_idx in range(query_group.repetitions): @@ -340,7 +349,9 @@ def handle_query_group( query_unix_time = aligned_eval_time( first_eval_time, query_group.repetition_delay_ms, repetition_idx ) - time.sleep(secs_until_send(query_unix_time, offset_secs, time.time())) + time.sleep( + secs_until_send(query_unix_time, lag_secs, offset_secs, time.time()) + ) logger.debug("Aligned evaluation time: {}", query_unix_time) else: if simulated_start_time is None: diff --git a/asap-tools/queriers/prometheus-client/tests/test_query_time.py b/asap-tools/queriers/prometheus-client/tests/test_query_time.py index 3ae121ab..ecb62ad2 100644 --- a/asap-tools/queriers/prometheus-client/tests/test_query_time.py +++ b/asap-tools/queriers/prometheus-client/tests/test_query_time.py @@ -8,43 +8,51 @@ secs_until_send, ) -# 2026-10-13 12:00:00 UTC, a multiple of 60 s. -MINUTE = 1791892800 +# 2026-10-13 10:00:00 UTC, a multiple of 300 s. +START = 1791885600 class FirstAlignedEvalTimeTest(unittest.TestCase): - def test_floors_to_the_interval_after_the_offset(self) -> None: - # Started at 12:00:35 with a 10 s offset: 12:00:25 floors to 12:00:00. - self.assertEqual(first_aligned_eval_time(MINUTE + 35.4, 60000, 10), MINUTE) - # Started at 12:00:05: the 12:00:00 window has not settled yet. - self.assertEqual(first_aligned_eval_time(MINUTE + 5, 60000, 10), MINUTE - 60) - - def test_one_second_interval(self) -> None: - self.assertEqual(first_aligned_eval_time(MINUTE + 7.9, 1000, 2), MINUTE + 5) + def test_first_multiple_after_the_starting_delay(self) -> None: + self.assertEqual(first_aligned_eval_time(START, 60000, 120), START + 120) + self.assertEqual(first_aligned_eval_time(START, 60000, 121), START + 180) + self.assertEqual(first_aligned_eval_time(START, 1000, 120), START + 120) + + def test_first_window_is_always_full(self) -> None: + # A starting delay shorter than the interval would evaluate a window + # that began before the data did. + self.assertEqual(first_aligned_eval_time(START, 300000, 120), START + 300) + self.assertEqual(first_aligned_eval_time(START, 60000, 0), START + 60) + + def test_start_off_the_interval_rejected(self) -> None: + # Eval times off the interval would miss ASAPQuery's aligned windows. + with self.assertRaisesRegex(ValueError, "not a multiple"): + first_aligned_eval_time(START + 60, 300000, 0) def test_sub_second_interval_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "whole-second"): - first_aligned_eval_time(MINUTE, 1500, 0) - - def test_negative_offset_rejected(self) -> None: - with self.assertRaisesRegex(ValueError, "query_time_offset"): - first_aligned_eval_time(MINUTE, 60000, -1) + first_aligned_eval_time(START, 1500, 0) class ScheduleTest(unittest.TestCase): def test_repetitions_step_by_the_interval(self) -> None: self.assertEqual( - [aligned_eval_time(MINUTE, 60000, k) for k in range(3)], - [MINUTE, MINUTE + 60, MINUTE + 120], + [aligned_eval_time(START, 60000, k) for k in range(3)], + [START, START + 60, START + 120], ) - def test_waits_until_offset_after_the_eval_time(self) -> None: - self.assertEqual(secs_until_send(MINUTE + 60, 10, MINUTE + 65), 5) + def test_waits_for_the_lag_and_offset_after_the_eval_time(self) -> None: + # Data stamped 10:01:00 is sent at 10:03:23; queried 10 s later. + self.assertEqual(secs_until_send(START + 60, 143, 10, START + 60), 153) def test_late_repetition_runs_at_once(self) -> None: # A slow repetition makes the next one late; it keeps its eval time # rather than skipping it. - self.assertEqual(secs_until_send(MINUTE + 60, 10, MINUTE + 75), 0) + self.assertEqual(secs_until_send(START + 60, 0, 10, START + 75), 0) + + def test_negative_offset_rejected(self) -> None: + with self.assertRaisesRegex(ValueError, "query_time_offset"): + secs_until_send(START, 0, -1, START) if __name__ == "__main__": From 06289b2a4de3b6884b1697894d95f494bc8ba781 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 14:07:52 -0400 Subject: [PATCH 9/9] fix(asap-tools): wait for an aligned feeder start instead of backdating Feeders now start at the next multiple of the aligned intervals' LCM (at most 600 s) rather than stamping data in the past, so the client sends each query at its eval time + query_time_offset and the feeder loses --send-at-ms. Also from review: - refuse sub-second aligned intervals and query_time_offset below ASAPQuery's 5 s allowed lateness - give feeder launches 10 s per node on top of the 5 s lead - keep feeders running under flow.no_teardown - refuse --sql-queries without --prebuilt-output-dir - refuse other exporters next to the feeder with no listed metrics Co-Authored-By: Claude Opus 5.5 --- .../data-sources/paced-feeder/README.md | 3 +- .../data-sources/paced-feeder/src/main.rs | 28 +--- .../CONFIG_PARAMETERS_REFERENCE.md | 10 +- asap-tools/experiments/experiment_run_e2e.py | 16 ++- .../experiments/experiment_utils/core.py | 13 +- .../experiment_utils/services/paced_feeder.py | 108 +++++++++------ .../sketch_bench_e2e_configs.py | 4 + .../experiments/tests/test_paced_feeder.py | 130 +++++++++++------- .../tests/test_sketch_bench_e2e_configs.py | 25 +++- .../prometheus-client/classes/query_time.py | 26 ++-- .../main_prometheus_client.py | 12 +- .../tests/test_query_time.py | 9 +- 12 files changed, 225 insertions(+), 159 deletions(-) diff --git a/asap-tools/data-sources/paced-feeder/README.md b/asap-tools/data-sources/paced-feeder/README.md index f5c1acbc..ab41bd70 100644 --- a/asap-tools/data-sources/paced-feeder/README.md +++ b/asap-tools/data-sources/paced-feeder/README.md @@ -160,8 +160,7 @@ http: 2000000 records/s ``` Event time starts at the next whole wall-clock second (or `--start-ms`), so -`NOW()`-relative queries see the data as it arrives. `--send-at-ms` sends the -first tick at a later wall-clock time, backdating the data by the difference. +`NOW()`-relative queries see the data as it arrives. ## Usage diff --git a/asap-tools/data-sources/paced-feeder/src/main.rs b/asap-tools/data-sources/paced-feeder/src/main.rs index 9c7b3eb2..1a106ba2 100644 --- a/asap-tools/data-sources/paced-feeder/src/main.rs +++ b/asap-tools/data-sources/paced-feeder/src/main.rs @@ -93,11 +93,6 @@ struct Args { /// next whole wall-clock second, so `NOW()`-relative queries see live data. #[arg(long)] start_ms: Option, - /// Wall-clock time to send the first tick, in ms since the epoch. Defaults - /// to `--start-ms`; a later time backdates the data by the difference, so - /// feeders launched at different moments share one event-time origin. - #[arg(long, requires = "start_ms")] - send_at_ms: Option, /// Rows per HTTP request, for gauges and counters rounded down to whole /// series (at least one). Defaults to 50,000 for remote write, which keeps @@ -139,7 +134,6 @@ struct RunStats { samples_per_sec: Option, seed: u64, start_ms: i64, - send_at_ms: i64, duration_secs: u64, target_rows_per_sec: usize, rows_sent: usize, @@ -279,14 +273,6 @@ fn next_whole_second_ms() -> i64 { (now_ms / 1_000 + 1) * 1_000 } -/// Data sent before its own timestamp would be in the future. -fn check_send_at(start_ms: i64, send_at_ms: i64) -> Result<()> { - if send_at_ms < start_ms { - bail!("--send-at-ms ({send_at_ms}) must not be before --start-ms ({start_ms})"); - } - Ok(()) -} - /// Sleep until wall-clock `epoch_ms`, then return the monotonic instant it maps to. async fn wait_for_wall_clock(epoch_ms: i64) -> Instant { let now_ms = SystemTime::now() @@ -452,8 +438,6 @@ async fn main() -> Result<()> { let sink = Arc::new(Sink::new(&args, sink_kind, url, &workload, batch_rows)?); let permits = Arc::new(Semaphore::new(args.concurrency)); let start_ms = args.start_ms.unwrap_or_else(next_whole_second_ms); - let send_at_ms = args.send_at_ms.unwrap_or(start_ms); - check_send_at(start_ms, send_at_ms)?; let metric_names = workload.names(); let series = if workload.metrics.is_empty() { @@ -466,12 +450,12 @@ async fn main() -> Result<()> { ) }; eprintln!( - "paced_feeder: {sink_kind:?} -> {url}, {}{series} = {} rows/s for {duration_secs}s, start_ms={start_ms}, send_at_ms={send_at_ms}", + "paced_feeder: {sink_kind:?} -> {url}, {}{series} = {} rows/s for {duration_secs}s, start_ms={start_ms}", metric_names.join(", "), workload.rows_per_tick(), ); - let base = wait_for_wall_clock(send_at_ms).await; + let base = wait_for_wall_clock(start_ms).await; let mut ticks = Vec::with_capacity(duration_secs as usize); let mut errors: Vec = Vec::new(); let mut failed_requests = 0; @@ -564,7 +548,6 @@ async fn main() -> Result<()> { samples_per_sec: (!workload.metrics.is_empty()).then_some(workload.samples_per_sec), seed: args.seed, start_ms, - send_at_ms, duration_secs, target_rows_per_sec: workload.rows_per_tick(), rows_sent, @@ -609,11 +592,4 @@ mod tests { assert!(validate_start_ms(1_700_000_000_000).is_ok()); assert!(validate_start_ms(1_700_000_000_500).is_err()); } - - #[test] - fn send_at_may_backdate_but_not_postdate_the_data() { - assert!(check_send_at(1_000, 1_000).is_ok()); - assert!(check_send_at(1_000, 301_000).is_ok()); - assert!(check_send_at(1_000, 999).is_err()); - } } diff --git a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md index 75addbc7..53bc78a8 100644 --- a/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md +++ b/asap-tools/experiments/CONFIG_PARAMETERS_REFERENCE.md @@ -543,13 +543,13 @@ table with `sketch_bench_e2e/sketch_bench_e2e_configs.py`. - **Usage**: Ensures data is available for lookback queries #### `experiment_params.query_groups[].client_options.align_to_interval` (boolean, optional) -- **Description**: Evaluate repetition k at `t_k = data_start_time + (max(1, ceil(starting_delay / T)) + k) * T`, with `T = repetition_delay_ms` (whole seconds), sending it at wall-clock `t_k + data_lag_secs + query_time_offset`. `starting_delay` counts in event time from `data_start_time`, not as a wall-clock sleep +- **Description**: Evaluate repetition k at `t_k = data_start_time + (max(1, ceil(starting_delay / T)) + k) * T`, with `T = repetition_delay_ms` (whole seconds), sending it at `t_k + query_time_offset`. `starting_delay` counts from `data_start_time`, not as a sleep from client start - **Default**: `false` -- **Usage**: Every evaluation time is a multiple of `T`, and repetition k covers the same samples in every mode however late the mode started; `query_time_offset` is how long the data of the window ending at `t_k` gets to arrive. Overrides the query engine's per-aggregation alignment in sketchdb mode. Requires `data_start_time` and `data_lag_secs` +- **Usage**: Every evaluation time is a multiple of `T`, and repetition k covers the same samples in every mode however late the mode started; `query_time_offset` (at least 5 s, ASAPQuery's allowed lateness) is how long the data of the window ending at `t_k` gets to arrive. Overrides the query engine's per-aggregation alignment in sketchdb mode. Requires `data_start_time`; the LCM of the aligned intervals must be at most 600 s -#### `experiment_params.query_groups[].client_options.data_start_time` / `data_lag_secs` (int / float) -- **Description**: Event time (epoch seconds) the paced feeder's data starts at, and how many seconds it runs behind the wall clock -- **Usage**: Not set by hand: `experiment_run_e2e.py` writes them into each mode's client config after starting the paced feeder, which backdates its data to the last multiple of the aligned groups' intervals +#### `experiment_params.query_groups[].client_options.data_start_time` (int) +- **Description**: Time (epoch seconds) the paced feeder's data starts at +- **Usage**: Not set by hand: `experiment_run_e2e.py` writes it into each mode's client config after starting the paced feeder, which waits for the next multiple of the aligned groups' intervals' LCM #### `experiment_params.query_groups[].client_options.starting_delay` (int, required) - **Description**: Initial delay in seconds before starting query execution diff --git a/asap-tools/experiments/experiment_run_e2e.py b/asap-tools/experiments/experiment_run_e2e.py index 8334a0a9..6653aad6 100644 --- a/asap-tools/experiments/experiment_run_e2e.py +++ b/asap-tools/experiments/experiment_run_e2e.py @@ -220,9 +220,15 @@ def main(cfg: DictConfig): forward_unsupported_queries=args.forward_unsupported_queries, auto_discover_labels=args.controller_auto_discover_labels, ) + # Also checks the aligned query groups before any service starts. + paced_feeder_align_secs = aligned_interval_lcm( + OmegaConf.to_container(cfg.experiment_params, resolve=True) + ) paced_feeder_service = PacedFeederService(provider, args) - # Otherwise a failed run leaves feeders pushing into the next run's sinks. - atexit.register(paced_feeder_service.stop) + if not args.no_teardown: + # Otherwise a failed run leaves feeders pushing into the next + # run's sinks. + atexit.register(paced_feeder_service.stop) for experiment_mode in experiment_modes: print(f"Running experiment mode: {experiment_mode}") @@ -536,14 +542,14 @@ def main(cfg: DictConfig): ) with open(local_client_config) as f: client_config = yaml.safe_load(f) - start_ms, send_at_ms = paced_feeder_service.start( + start_ms = paced_feeder_service.start( config=paced_feeder_config, sinks=sinks, experiment_output_dir=experiment_output_dir, local_experiment_dir=local_experiment_dir, - align_secs=aligned_interval_lcm(client_config), + align_secs=paced_feeder_align_secs, ) - set_data_start(client_config, start_ms, send_at_ms) + set_data_start(client_config, start_ms) with open(local_client_config, "w") as f: yaml.safe_dump(client_config, f, sort_keys=False) sync.rsync_controller_client_configs( diff --git a/asap-tools/experiments/experiment_utils/core.py b/asap-tools/experiments/experiment_utils/core.py index 2a840130..8882b831 100644 --- a/asap-tools/experiments/experiment_utils/core.py +++ b/asap-tools/experiments/experiment_utils/core.py @@ -197,7 +197,10 @@ def call_generate_prometheus_config( import generate_prometheus_config import generate_victoriametrics_config - from experiment_utils.services.paced_feeder import paced_feeder_metric_names + from experiment_utils.services.paced_feeder import ( + PACED_FEEDER_EXPORTER, + paced_feeder_metric_names, + ) # The paced feeder writes its metrics to the query engine itself, so # Prometheus forwarding them too would send them twice. @@ -210,6 +213,14 @@ def call_generate_prometheus_config( # With only fed metrics there is nothing for Prometheus to forward. (An # empty list without the feeder keeps meaning "forward everything".) forward_to_query_engine = bool(metrics_to_remote_write) or not fed_metrics + exporter_list = cfg.experiment_params.get("exporters", {}).get("exporter_list", {}) + other_exporters = sorted(set(exporter_list) - {PACED_FEEDER_EXPORTER}) + if not forward_to_query_engine and other_exporters: + raise ValueError( + f"exporters {other_exporters} run next to the paced feeder but none " + "of their metrics are listed under experiment_params.metrics, so " + "Prometheus would forward nothing of theirs to the query engine" + ) # Set up remote write if this is a SketchDB experiment if ( experiment_mode == sketchdb_experiment_name diff --git a/asap-tools/experiments/experiment_utils/services/paced_feeder.py b/asap-tools/experiments/experiment_utils/services/paced_feeder.py index 7953946f..cc9da43c 100644 --- a/asap-tools/experiments/experiment_utils/services/paced_feeder.py +++ b/asap-tools/experiments/experiment_utils/services/paced_feeder.py @@ -31,12 +31,20 @@ PACED_FEEDER_BINARY = "./target/release/paced_feeder" # The brackets keep pgrep/pkill from matching the shell that runs them. PACED_FEEDER_PROCESS_PATTERN = "[t]arget/release/paced_feeder" -# Feeders are launched over SSH, one node after another, and all send their -# first tick this long after the launches begin. +# Feeders are launched over SSH, one node after another, and all start at the +# first aligned time at least this long after the launches begin: a base plus, +# per node, enough for one SSH retry (run_cmd_with_retry waits 5 s). PACED_FEEDER_START_LEAD_SECS = 5 +PACED_FEEDER_LAUNCH_SECS_PER_NODE = 10 # A feeder stops after its first tick if every request fails (e.g. a wrong # URL), so check the processes one tick after the shared start. -PACED_FEEDER_STARTUP_CHECK_SECS = PACED_FEEDER_START_LEAD_SECS + 2 +PACED_FEEDER_STARTUP_CHECK_SECS = 2 +# Feeders wait for the next multiple of the aligned intervals' LCM before +# sending; refuse intervals that would make that wait longer than this. +PACED_FEEDER_MAX_ALIGN_SECS = 600 +# ASAPQuery's default allowed lateness: a window closes this long after its +# end, so an aligned query must not be sent sooner. +QUERY_ENGINE_ALLOWED_LATENESS_SECS = 5 SINK_QUERY_ENGINE = "query_engine" SINK_PROMETHEUS = "prometheus" @@ -49,36 +57,55 @@ def paced_feeder_metric_names(experiment_params) -> List[str]: return list(feeder["metrics"]) if feeder is not None else [] -def backdated_start_ms(send_at_ms: int, align_secs: int) -> int: - """The last multiple of `align_secs` at or before `send_at_ms`: the feeder - stamps its data from there, so every mode's data starts on the same - boundary of each aligned query group's interval.""" +def aligned_start_ms(earliest_ms: int, align_secs: int) -> int: + """The first multiple of `align_secs` at or after `earliest_ms`: every + mode's data starts on the same boundary of each aligned query group's + interval.""" align_ms = align_secs * 1000 - return send_at_ms // align_ms * align_ms + return -(-earliest_ms // align_ms) * align_ms def aligned_interval_lcm(client_config: Dict[str, Any]) -> int: """LCM, in seconds, of the repeat intervals of the client config's - `align_to_interval` groups; 1 if there are none.""" - intervals = [ - g["repetition_delay_ms"] // 1000 - for g in client_config["query_groups"] - if g.get("client_options", {}).get("align_to_interval") - ] + `align_to_interval` groups (1 if there are none), after checking that + those groups can be aligned.""" + intervals = [] + for group in client_config["query_groups"]: + options = group.get("client_options", {}) + if not options.get("align_to_interval"): + continue + interval_ms = group["repetition_delay_ms"] + if interval_ms <= 0 or interval_ms % 1000 != 0: + raise ValueError( + f"query group {group['id']}: align_to_interval needs a " + f"whole-second repetition_delay_ms, got {interval_ms}" + ) + offset = options.get("query_time_offset", 0) + if offset < QUERY_ENGINE_ALLOWED_LATENESS_SECS: + raise ValueError( + f"query group {group['id']}: align_to_interval needs " + f"query_time_offset >= {QUERY_ENGINE_ALLOWED_LATENESS_SECS} " + f"(ASAPQuery's allowed lateness), got {offset}" + ) + intervals.append(interval_ms // 1000) # math.lcm needs Python 3.9. - return functools.reduce(lambda a, b: a * b // math.gcd(a, b), intervals, 1) + lcm = functools.reduce(lambda a, b: a * b // math.gcd(a, b), intervals, 1) + if lcm > PACED_FEEDER_MAX_ALIGN_SECS: + raise ValueError( + f"aligned repeat intervals {sorted(set(intervals))} s have an LCM of " + f"{lcm} s; the feeder would wait up to that long to start " + f"(limit {PACED_FEEDER_MAX_ALIGN_SECS} s)" + ) + return lcm -def set_data_start( - client_config: Dict[str, Any], start_ms: int, send_at_ms: int -) -> None: - """Tell the client's aligned groups when the data starts in event time and - how far it runs behind the wall clock (see the client's query_time.py).""" +def set_data_start(client_config: Dict[str, Any], start_ms: int) -> None: + """Tell the client's aligned groups when the data starts (see the + client's query_time.py).""" for group in client_config["query_groups"]: options = group.get("client_options", {}) if options.get("align_to_interval"): options["data_start_time"] = start_ms // 1000 - options["data_lag_secs"] = (send_at_ms - start_ms) / 1000 def feeds_prometheus(mode: Dict[str, Any]) -> bool: @@ -93,8 +120,8 @@ def paced_feeder_min_duration_secs( ) -> float: """Seconds of data that last until the final query: queries end at most startup check + steady_state_wait + the query groups' running time after - launch. Aligned groups need less (their last evaluation is at most the - starting delay + running time into the data), so this covers both.""" + the data starts. Aligned groups need less (their last evaluation is at + most the starting delay + running time into the data).""" return PACED_FEEDER_STARTUP_CHECK_SECS + steady_state_wait + query_running_secs @@ -187,15 +214,13 @@ def paced_feeder_command( metric: str, metric_type: str, start_ms: int, - send_at_ms: int, stats_out: str, ) -> str: cmd = ( f"{PACED_FEEDER_BINARY} --sink remote-write --url {url}" f" --workload-table {workload_table} --metric {metric}" f" --seed {config['seed']} --duration-secs {config['duration_secs']}" - f" --start-ms {start_ms} --send-at-ms {send_at_ms}" - f" --stats-out {stats_out}" + f" --start-ms {start_ms} --stats-out {stats_out}" ) # The feeder refuses the other kind's rate flag. if metric_type == RECORDS_METRIC_TYPE: @@ -231,9 +256,9 @@ def start( local_experiment_dir: str, align_secs: int = 1, **kwargs, - ) -> Tuple[int, int]: - """Launch the feeders; returns the data's event-time start and the - wall-clock time its first tick is sent, both in epoch ms.""" + ) -> int: + """Launch the feeders; returns when their data starts, in epoch ms, + once they are running.""" output_dir = os.path.join(experiment_output_dir, "paced_feeder_output") metric_types = table_metric_types(config["workload_table"]) jobs = [(sink, metric) for sink in sinks for metric in config["metrics"]] @@ -256,10 +281,15 @@ def start( self.provider, config["workload_table"], output_dir, node_idx ) - # One shared send time for every process of this mode, taken after the - # SSH prep above so the lead covers only the launches. - send_at_ms = (math.ceil(time.time()) + PACED_FEEDER_START_LEAD_SECS) * 1000 - start_ms = backdated_start_ms(send_at_ms, align_secs) + # One shared start for every process of this mode, taken after the SSH + # prep above so the lead covers only the launches. + lead_secs = ( + PACED_FEEDER_START_LEAD_SECS + + PACED_FEEDER_LAUNCH_SECS_PER_NODE * len(used_nodes) + ) + start_ms = aligned_start_ms( + (math.ceil(time.time()) + lead_secs) * 1000, align_secs + ) node_cmds: List[Tuple[int, str]] = [] for node_idx, (sink, metric) in zip(job_nodes, jobs): name = f"{sink}_{metric}" @@ -270,7 +300,6 @@ def start( metric, metric_types[metric], start_ms, - send_at_ms, os.path.join(output_dir, f"{name}_stats.json"), ) node_cmds.append( @@ -285,7 +314,6 @@ def start( with open(os.path.join(local_config_dir, "cmds.sh"), "w") as f: f.write("\n".join(cmd for _, cmd in node_cmds) + "\n") - # One SSH per node, so the launches fit in the lead. for node_idx in used_nodes: self.provider.execute_command( node_idx=node_idx, @@ -295,16 +323,16 @@ def start( popen=False, ignore_errors=False, ) - if time.time() * 1000 > send_at_ms: - # A feeder launched after its send time starts late, so its data + if time.time() * 1000 > start_ms: + # A feeder launched after the start begins late, so its data # would trail the other sinks'. self.stop() raise RuntimeError( f"paced_feeder: launching on {len(used_nodes)} nodes took longer " - f"than the {PACED_FEEDER_START_LEAD_SECS} s lead" + f"than the {lead_secs} s lead" ) - time.sleep(PACED_FEEDER_STARTUP_CHECK_SECS) + time.sleep(start_ms / 1000 - time.time() + PACED_FEEDER_STARTUP_CHECK_SECS) for node_idx in used_nodes: expected = job_nodes.count(node_idx) # No $(...): the SSH command is double-quoted on this side. @@ -321,7 +349,7 @@ def start( f"paced_feeder: expected {expected} processes running on " f"node {node_idx}; see the .out files in {output_dir}" ) - return start_ms, send_at_ms + return start_ms def stop(self, **kwargs) -> None: self.provider.execute_command_parallel( diff --git a/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py b/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py index 9bfec0cf..9e6b487d 100644 --- a/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py +++ b/asap-tools/experiments/sketch_bench_e2e/sketch_bench_e2e_configs.py @@ -93,6 +93,10 @@ def build_config( sql_queries_file: Optional[str] = None, ) -> Tuple[dict, List[str]]: """Return (config, notes on what was skipped).""" + if sql_queries_file is not None and prebuilt_output_dir is None: + # The planner rejects the SQL groups' `time_window_seconds` and can't + # plan them; they need configs from `asap-planner --planner saved`. + raise ValueError("--sql-queries needs --prebuilt-output-dir") e2e = load_e2e(table_path) notes: List[str] = [] diff --git a/asap-tools/experiments/tests/test_paced_feeder.py b/asap-tools/experiments/tests/test_paced_feeder.py index f3409afb..c297519a 100644 --- a/asap-tools/experiments/tests/test_paced_feeder.py +++ b/asap-tools/experiments/tests/test_paced_feeder.py @@ -15,11 +15,12 @@ from experiment_utils.services.docker_prometheus import DockerPrometheusService from experiment_utils.services import paced_feeder from experiment_utils.services.paced_feeder import ( + PACED_FEEDER_LAUNCH_SECS_PER_NODE, PACED_FEEDER_START_LEAD_SECS, PACED_FEEDER_STARTUP_CHECK_SECS, PacedFeederService, aligned_interval_lcm, - backdated_start_ms, + aligned_start_ms, paced_feeder_min_duration_secs, paced_feeder_command, paced_feeder_sinks, @@ -107,7 +108,6 @@ def test_command_line(self): "data", "gauge", 1700000000000, - 1700000005000, "/out/query_engine_data_stats.json", ) self.assertEqual( @@ -115,7 +115,6 @@ def test_command_line(self): "./target/release/paced_feeder --sink remote-write" f" --url {QE_URL} --workload-table /out/table.json --metric data" " --seed 42 --duration-secs 600 --start-ms 1700000000000" - " --send-at-ms 1700000005000" " --stats-out /out/query_engine_data_stats.json", ) @@ -127,7 +126,6 @@ def test_optional_flags_are_appended(self): "data", "gauge", 0, - 0, "/s", ) self.assertTrue(cmd.endswith(" --samples-per-sec 20 --concurrency 4")) @@ -140,7 +138,7 @@ def test_each_metric_gets_its_own_kind_of_rate(self): def rate_flags(metric, metric_type): cmd = paced_feeder_command( - config, "/t", QE_URL, metric, metric_type, 0, 0, "/s" + config, "/t", QE_URL, metric, metric_type, 0, "/s" ) return [f for f in cmd.split() if f.endswith("-per-sec")] @@ -255,10 +253,10 @@ def test_feeder_ending_before_the_queries(self): check_paced_feeder_config(feeder_config(self.table), BOTH_MODES, 1, 601) def test_duration_covers_startup_wait_and_queries(self): - # 60 s steady_state_wait and 533 s of queries: the feeder's startup + # 60 s steady_state_wait and 538 s of queries: the feeder's startup # check comes on top, so 600 s is exactly enough and 599 s is not. - min_secs = paced_feeder_min_duration_secs(60, 533) - self.assertEqual(min_secs, PACED_FEEDER_STARTUP_CHECK_SECS + 60 + 533) + min_secs = paced_feeder_min_duration_secs(60, 538) + self.assertEqual(min_secs, PACED_FEEDER_STARTUP_CHECK_SECS + 60 + 538) self.assertEqual(min_secs, 600) check_paced_feeder_config(feeder_config(self.table), BOTH_MODES, 1, min_secs) with self.assertRaisesRegex(ValueError, "at least 600"): @@ -280,46 +278,62 @@ def test_missing_keys_rejected_with_the_other_exporters(self): self.assertIn("duration_secs", reason) -def client_config(*intervals_secs, aligned=True): +def client_config(*intervals_ms, aligned=True, offset=10): return { "query_groups": [ { "id": i, - "repetition_delay_ms": t * 1000, - "client_options": {"repetitions": 2, "align_to_interval": aligned}, + "repetition_delay_ms": t, + "client_options": { + "repetitions": 2, + "align_to_interval": aligned, + "query_time_offset": offset, + }, } - for i, t in enumerate(intervals_secs) + for i, t in enumerate(intervals_ms) ] } class DataStartTest(unittest.TestCase): - def test_backdates_to_the_last_boundary(self): - # Sent from 10:02:23, stamped from 10:00:00. - send_at = 1791885743000 - self.assertEqual(backdated_start_ms(send_at, 300), 1791885600000) - self.assertEqual(backdated_start_ms(1791885600000, 300), 1791885600000) - self.assertEqual(backdated_start_ms(send_at, 1), send_at) - - def test_origin_fits_every_aligned_interval(self): - self.assertEqual(aligned_interval_lcm(client_config(1, 60, 300)), 300) - self.assertEqual(aligned_interval_lcm(client_config(60, 90)), 180) - self.assertEqual(aligned_interval_lcm(client_config(60, aligned=False)), 1) + def test_waits_for_the_next_boundary(self): + # Ready at 10:02:23, starts at 10:05:00. + self.assertEqual(aligned_start_ms(1791885743000, 300), 1791885900000) + self.assertEqual(aligned_start_ms(1791885600000, 300), 1791885600000) + self.assertEqual(aligned_start_ms(1791885743000, 1), 1791885743000) + + def test_start_fits_every_aligned_interval(self): + lcm = aligned_interval_lcm + self.assertEqual(lcm(client_config(1000, 60000, 300000)), 300) + self.assertEqual(lcm(client_config(60000, 90000)), 180) + self.assertEqual(lcm(client_config(60000, aligned=False)), 1) + + def test_long_wait_refused(self): + # 420 s and 3600 s intervals would wait up to 7 h for an aligned start. + with self.assertRaisesRegex(ValueError, "LCM of 25200 s"): + aligned_interval_lcm(client_config(420000, 3600000)) + + def test_sub_second_interval_refused(self): + # Truncating to whole seconds would give an LCM of 0 (or misalign 1.5 s). + for interval_ms in (500, 1500): + with self.assertRaisesRegex(ValueError, "whole-second"): + aligned_interval_lcm(client_config(interval_ms)) + + def test_offset_below_the_allowed_lateness_refused(self): + # Sent sooner, a query reads an ASAPQuery window that is still open. + with self.assertRaisesRegex(ValueError, "query_time_offset >= 5"): + aligned_interval_lcm(client_config(60000, offset=0)) + aligned_interval_lcm(client_config(60000, offset=5)) def test_aligned_groups_learn_the_data_start(self): - config = client_config(60) + config = client_config(60000) config["query_groups"].append( {"id": 9, "repetition_delay_ms": 1000, "client_options": {}} ) - set_data_start(config, 1791885600000, 1791885743000) + set_data_start(config, 1791885600000) self.assertEqual( - config["query_groups"][0]["client_options"], - { - "repetitions": 2, - "align_to_interval": True, - "data_start_time": 1791885600, - "data_lag_secs": 143.0, - }, + config["query_groups"][0]["client_options"]["data_start_time"], + 1791885600, ) self.assertEqual(config["query_groups"][1]["client_options"], {}) @@ -414,30 +428,38 @@ def flag_values(commands, flag): if "--sink" in c } - def test_send_time_leads_the_launches_not_the_prep(self): + # Lead for the 2 worker nodes the tests launch on. + LEAD_SECS = PACED_FEEDER_START_LEAD_SECS + 2 * PACED_FEEDER_LAUNCH_SECS_PER_NODE + + def test_start_leads_the_launches_not_the_prep(self): # The SSH prep (2 nodes x binary check + table copy) takes 40 fake - # seconds; the shared send time must still be the lead after it. + # seconds; the shared start must still be the lead after it. commands = self.start(running_count=2) - send_at_ms = (1700000000 + 4 * PREP_SECS + PACED_FEEDER_START_LEAD_SECS) * 1000 - self.assertEqual(self.flag_values(commands, "--send-at-ms"), {send_at_ms}) - self.assertEqual(self.flag_values(commands, "--start-ms"), {send_at_ms}) - self.assertEqual(self.started, (send_at_ms, send_at_ms)) + start_ms = (1700000000 + 4 * PREP_SECS + self.LEAD_SECS) * 1000 + self.assertEqual(self.flag_values(commands, "--start-ms"), {start_ms}) + self.assertEqual(self.started, start_ms) - def test_data_is_backdated_to_the_aligned_boundary(self): - # Sent from 1700000045 s, stamped from the last multiple of 300 s. + def test_start_waits_for_the_aligned_boundary(self): + # Ready at 1700000065 s; the next multiple of 300 s is 1700000100 s. commands = self.start(running_count=2, align_secs=300) - self.assertEqual(self.flag_values(commands, "--start-ms"), {1699999800000}) - self.assertEqual(self.flag_values(commands, "--send-at-ms"), {1700000045000}) - self.assertEqual(self.started, (1699999800000, 1700000045000)) + self.assertEqual(self.flag_values(commands, "--start-ms"), {1700000100000}) + self.assertEqual(self.started, 1700000100000) def test_one_launch_per_node(self): self.start(running_count=2) self.assertEqual(self.provider.launches, 2) + def test_launches_get_time_per_node(self): + # A retried SSH (5 s) on each node still fits. + self.start(running_count=2, launch_secs=PACED_FEEDER_LAUNCH_SECS_PER_NODE) + def test_launch_slower_than_the_lead_fails(self): - # A feeder launched after its send time would trail the other sinks. - with self.assertRaisesRegex(RuntimeError, "longer than the 5 s lead"): - self.start(running_count=2, launch_secs=3) + # A feeder launched after the start would trail the other sinks. + per_node = self.LEAD_SECS // 2 + 1 + with self.assertRaisesRegex( + RuntimeError, f"longer than the {self.LEAD_SECS} s lead" + ): + self.start(running_count=2, launch_secs=per_node) self.assertTrue(any("pkill" in c for _, c in self.provider.commands)) def test_records_and_gauge_processes_get_their_rate_flags(self): @@ -560,7 +582,7 @@ class PrometheusForwardingTest(unittest.TestCase): """In sketchdb mode Prometheus must not forward fed metrics to the query engine, which the feeder already writes to.""" - def remote_write_args(self, metrics, fed=("data",)): + def remote_write_args(self, metrics, fed=("data",), other_exporters=()): cfg = OmegaConf.create( { "streaming": { @@ -574,7 +596,10 @@ def remote_write_args(self, metrics, fed=("data",)): "experiment_params": { "exporters": { "only_start_if_queries_exist": False, - "exporter_list": {"paced_feeder": {"metrics": list(fed)}}, + "exporter_list": { + "paced_feeder": {"metrics": list(fed)}, + **{name: {} for name in other_exporters}, + }, }, "query_groups": [], "metrics": metrics, @@ -599,6 +624,15 @@ def test_only_fed_metrics_means_no_remote_write(self): ) self.assertIsNone(args.remote_write_url) + def test_unlisted_exporter_next_to_the_feeder_refused(self): + # With nothing left to forward, remote write is off, so the other + # exporter's metrics would never reach ASAPQuery. + with self.assertRaisesRegex(ValueError, r"\['node_exporter'\]"): + self.remote_write_args( + [{"metric": "data", "labels": ["job"], "exporter": "paced_feeder"}], + other_exporters=["node_exporter"], + ) + def test_scraped_metrics_are_still_forwarded(self): args = self.remote_write_args( [ diff --git a/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py b/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py index 15dafe76..fc38e390 100644 --- a/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py +++ b/asap-tools/experiments/tests/test_sketch_bench_e2e_configs.py @@ -72,6 +72,12 @@ def tearDown(self): self.tmp.cleanup() def build(self, **kwargs): + if "sql_queries_file" in kwargs and "prebuilt_output_dir" not in kwargs: + prebuilt = os.path.join(self.tmp.name, "sql_prebuilt") + os.makedirs(prebuilt, exist_ok=True) + for name in experiment_config.PLANNER_OUTPUT_FILES: + open(os.path.join(prebuilt, name), "w").close() + kwargs["prebuilt_output_dir"] = prebuilt return sbe.build_config( self.table, seed=7, @@ -203,6 +209,17 @@ def test_query_phase_shorter_than_an_interval(self): self.table, seed=7, starting_delay_secs=0, query_duration_secs=30 ) + def test_sql_queries_need_prebuilt_planner_configs(self): + # The planner rejects the SQL groups' time_window_seconds. + with self.assertRaisesRegex(ValueError, "--prebuilt-output-dir"): + sbe.build_config( + self.table, + seed=7, + starting_delay_secs=120, + query_duration_secs=600, + sql_queries_file=os.path.join(self.tmp.name, "q.sql"), + ) + def test_prebuilt_output_dir(self): prebuilt = os.path.join(self.tmp.name, "prebuilt") os.makedirs(prebuilt) @@ -257,13 +274,13 @@ def test_runner_can_anchor_the_generated_client_config(self): with open(path) as f: client_config = yaml.safe_load(f) self.assertEqual(aligned_interval_lcm(client_config), 60) - set_data_start(client_config, 1791885600000, 1791885743000) + set_data_start(client_config, 1791885600000) self.assertEqual( { - (o["data_start_time"], o["data_lag_secs"]) - for o in (g["client_options"] for g in client_config["query_groups"]) + g["client_options"]["data_start_time"] + for g in client_config["query_groups"] }, - {(1791885600, 143.0)}, + {1791885600}, ) def test_generated_config_with_records_validates(self): diff --git a/asap-tools/queriers/prometheus-client/classes/query_time.py b/asap-tools/queriers/prometheus-client/classes/query_time.py index d42cf852..b35c5d78 100644 --- a/asap-tools/queriers/prometheus-client/classes/query_time.py +++ b/asap-tools/queriers/prometheus-client/classes/query_time.py @@ -1,20 +1,18 @@ """Evaluation times aligned to a query group's repeat interval. -With `client_options.align_to_interval`, the data starts at event time S +With `client_options.align_to_interval`, the data starts at time S (`client_options.data_start_time`, epoch seconds, a multiple of every aligned -group's interval) and runs L seconds behind the wall clock -(`client_options.data_lag_secs`). Repetition k of a group with repeat interval -T (`repetition_delay_ms`), starting delay D (`starting_delay`) and settle -offset O (`query_time_offset`, default 0) evaluates at +group's interval). Repetition k of a group with repeat interval T +(`repetition_delay_ms`), starting delay D (`starting_delay`) and settle offset +O (`query_time_offset`, default 0) evaluates at t_k = S + (max(1, ceil(D / T)) + k) * T -and is sent at wall-clock time t_k + L + O, once its data has arrived. Every -evaluation time is a multiple of T, matching ASAPQuery's epoch-aligned windows, -and repetition k covers the same samples in every run regardless of when the -run started. For example, T = 60 s, D = 120 s, O = 10 s, and data stamped from -10:00:00 but sent from 10:02:23 (L = 143 s): repetitions evaluate at 10:02:00, -10:03:00, ... and are sent at 10:04:33, 10:05:33, ... +and is sent at t_k + O, once its data has arrived. Every evaluation time is a +multiple of T, matching ASAPQuery's epoch-aligned windows, and repetition k +covers the same samples in every run regardless of when the run started. For +example, T = 60 s, D = 120 s, O = 10 s and data from 10:00:00: repetitions +evaluate at 10:02:00, 10:03:00, ... and are sent at 10:02:10, 10:03:10, ... """ import math @@ -45,10 +43,8 @@ def aligned_eval_time(first_eval_time: int, interval_ms: int, repetition: int) - return first_eval_time + repetition * (interval_ms // MS_PER_SEC) -def secs_until_send( - eval_time: int, data_lag_secs: float, offset_secs: float, now_secs: float -) -> float: +def secs_until_send(eval_time: int, offset_secs: float, now_secs: float) -> float: """How long to wait before evaluating at `eval_time`; 0 when already late.""" if offset_secs < 0: raise ValueError(f"query_time_offset must be >= 0, got {offset_secs}") - return max(0.0, eval_time + data_lag_secs + offset_secs - now_secs) + return max(0.0, eval_time + offset_secs - now_secs) diff --git a/asap-tools/queriers/prometheus-client/main_prometheus_client.py b/asap-tools/queriers/prometheus-client/main_prometheus_client.py index a50cecb1..1382c7c3 100644 --- a/asap-tools/queriers/prometheus-client/main_prometheus_client.py +++ b/asap-tools/queriers/prometheus-client/main_prometheus_client.py @@ -332,13 +332,11 @@ def handle_query_group( ) if align_to_interval: data_start_time = getattr(query_group, "data_start_time", None) - data_lag_secs = getattr(query_group, "data_lag_secs", None) - if data_start_time is None or data_lag_secs is None: + if data_start_time is None: raise ValueError( - "align_to_interval needs client_options.data_start_time and " - "data_lag_secs (set by the paced feeder runner)" + "align_to_interval needs client_options.data_start_time " + "(set by the paced feeder runner)" ) - lag_secs = float(data_lag_secs) offset_secs = getattr(query_group, "query_time_offset", 0) first_eval_time = first_aligned_eval_time( data_start_time, query_group.repetition_delay_ms, query_group.starting_delay @@ -349,9 +347,7 @@ def handle_query_group( query_unix_time = aligned_eval_time( first_eval_time, query_group.repetition_delay_ms, repetition_idx ) - time.sleep( - secs_until_send(query_unix_time, lag_secs, offset_secs, time.time()) - ) + time.sleep(secs_until_send(query_unix_time, offset_secs, time.time())) logger.debug("Aligned evaluation time: {}", query_unix_time) else: if simulated_start_time is None: diff --git a/asap-tools/queriers/prometheus-client/tests/test_query_time.py b/asap-tools/queriers/prometheus-client/tests/test_query_time.py index ecb62ad2..acac7c1c 100644 --- a/asap-tools/queriers/prometheus-client/tests/test_query_time.py +++ b/asap-tools/queriers/prometheus-client/tests/test_query_time.py @@ -41,18 +41,17 @@ def test_repetitions_step_by_the_interval(self) -> None: [START, START + 60, START + 120], ) - def test_waits_for_the_lag_and_offset_after_the_eval_time(self) -> None: - # Data stamped 10:01:00 is sent at 10:03:23; queried 10 s later. - self.assertEqual(secs_until_send(START + 60, 143, 10, START + 60), 153) + def test_waits_for_the_offset_after_the_eval_time(self) -> None: + self.assertEqual(secs_until_send(START + 60, 10, START + 65), 5) def test_late_repetition_runs_at_once(self) -> None: # A slow repetition makes the next one late; it keeps its eval time # rather than skipping it. - self.assertEqual(secs_until_send(START + 60, 0, 10, START + 75), 0) + self.assertEqual(secs_until_send(START + 60, 10, START + 75), 0) def test_negative_offset_rejected(self) -> None: with self.assertRaisesRegex(ValueError, "query_time_offset"): - secs_until_send(START, 0, -1, START) + secs_until_send(START, -1, START) if __name__ == "__main__":