diff --git a/.design_docs/accelerated-random-workload-eval.md b/.design_docs/accelerated-random-workload-eval.md index e8983734..4d3ac665 100644 --- a/.design_docs/accelerated-random-workload-eval.md +++ b/.design_docs/accelerated-random-workload-eval.md @@ -5,7 +5,7 @@ Run a practical, end-to-end version of the paper's §6.2 experiment before the full PR 862 workload. The experiment should retain the important workload shape—many input series and few returned series—and show the benefit of -planning a *workload* of related RQEs rather than isolated queries. +planning a *workload* of related repeating query expressions (RQEs) rather than isolated queries. The experiment has ten deterministic random workloads. Each workload contains exactly ten atomic user queries; binary arithmetic queries are out of scope for @@ -51,20 +51,21 @@ unbounded. | Lookback `W` | Allowed repeat intervals `T` | | ---: | ---: | -| 100 ms | 1 s | +| 100 s | 1 s | | 1 min | 1 s, 2 s | | 5 min | 5 s, 10 s | | 10 min | 10 s, 20 s | | 20 min | 30 s, 60 s | The maximum accelerated backfill is therefore only 20 minutes of event-time -history. The first 100 ms class is for spatial queries; temporal queries use -the 1--20 minute classes. The spatial class targets a 100 ms repetition -interval. Current exact-MV support is whole-second only, so this class is -gated on [#876](https://github.com/ProjectASAP/ASAPQuery/issues/876), which -adds millisecond interval metadata, `DateTime64` bucket flooring and edge -handling, a sub-second query schedule, and raw-versus-MV equivalence tests. -This is a limitation of the current baseline generator, not of ClickHouse. +history. The first 100 s class is for quantile queries; the other temporal +queries use the 1--20 minute classes. Scraping remains every 100 ms, so each +quantile query consumes 1,000 samples per leaf series: the first N that +sketch-bench's accuracy curves cover, below which no saved plan can serve it. +Its whole-second repetition and lookback are compatible with the current +exact-MV arm; it is not gated on +[#876](https://github.com/ProjectASAP/ASAPQuery/issues/876). That issue is +needed only for a future sub-second query/MV schedule. ## Query-template catalogue @@ -119,12 +120,6 @@ first pilot workload. ### Quantile -For the 100 ms spatial class, PromQL has exact grouped semantics: - -```promql -quantile by (G) (phi, data) -``` - For a temporal grouped quantile, PromQL cannot directly express a pooled quantile over every raw sample in each group. The PromQL-compatible rendering is instead the average of each leaf series' temporal quantile: @@ -182,6 +177,13 @@ The manifest also contains the label hierarchy, cardinalities of every legal grouping, value/key distribution facts, feeder seed, and support class (`common-promql`, `counter-gated-promql`, or `sql-only`). +The `data_facts` contract fixes a 100 ms scrape cadence and 20-minute +backfill, a deterministic Zipf-like ranking of the 10,000 leaf series, a +rank-scaled Pareto gauge and monotonic counter (no resets), plus a 10,000-ID user +universe. Each leaf rotates through 1,000 deterministically assigned IDs; +the pools overlap across leaves. The eventual feeder and every baseline +adapter must implement these facts rather than choose their own distributions. + Sampling is constrained rather than purely independent: - exactly ten atomic queries per workload and no binary queries; diff --git a/asap-planner-rs/src/optimizer/aqe_extractor.rs b/asap-planner-rs/src/optimizer/aqe_extractor.rs index f7f444ff..e755be57 100644 --- a/asap-planner-rs/src/optimizer/aqe_extractor.rs +++ b/asap-planner-rs/src/optimizer/aqe_extractor.rs @@ -10,7 +10,7 @@ use promql_utilities::query_logics::enums::Statistic; use crate::config::input::TableDefinition; use crate::planner::patterns::build_patterns; -use crate::planner::promql::{parse_binary_arms, BinaryArm}; +use crate::planner::promql::{parse_binary_arms, strip_parens, BinaryArm}; use crate::planner::sql::{parse_sql_query, sql_data_range_ms, sql_statistics}; use super::error::OptimizerError; @@ -261,21 +261,39 @@ pub(crate) fn decompose_to_leaves(query: &str) -> Vec { leaves } -/// Whether any leaf of `query` is an `avg` that `decompose_to_leaves` rewrites. +/// Whether `query` has an `avg` the engine can't answer: one that +/// `decompose_to_leaves` splits into sum and count, or a per-series avg in a +/// binary arm, which runs without its query-time avg. pub(crate) fn contains_avg(query: &str) -> bool { match parse_binary_arms(query) { - Some((lhs, rhs)) => [lhs, rhs] - .into_iter() - .any(|arm| matches!(arm, BinaryArm::Query(q) if contains_avg(&q))), + Some((lhs, rhs)) => [lhs, rhs].into_iter().any(|arm| { + matches!(arm, BinaryArm::Query(q) if contains_avg(&q) || per_series_avg(&q).is_some()) + }), None => rewrite_avg(query).len() > 1, } } +/// `avg by (l) (f(x[w]))` of a per-series function `f`: the leaf +/// `f(x[w])`, and the avg that averages its series at query time. +pub(crate) fn per_series_avg(query: &str) -> Option<(String, AggregateExpr)> { + match promql_parser::parser::parse(query).ok()? { + Expr::Aggregate(agg) if agg.op.id() == token::T_AVG => match strip_parens(&agg.expr) { + call @ Expr::Call(_) => Some((call.to_string(), agg)), + _ => None, + }, + _ => None, + } +} + /// Rewrite an `avg` leaf into the sum and count leaves it is computed from: /// `avg by (l) (x)` → `sum by (l) (x)`, `count by (l) (x)`, and /// `avg_over_time(x[5m])` → `sum_over_time(x[5m])`, `count_over_time(x[5m])`. -/// Any other query is returned unchanged. +/// A per-series avg is its leaf (see `per_series_avg`). Any other query is +/// returned unchanged. fn rewrite_avg(query: &str) -> Vec { + if let Some((leaf, _)) = per_series_avg(query) { + return vec![leaf]; + } match promql_parser::parser::parse(query) { Ok(Expr::Aggregate(agg)) if agg.op.id() == token::T_AVG => [token::T_SUM, token::T_COUNT] .into_iter() @@ -608,6 +626,23 @@ mod tests { assert!(items.iter().all(|i| i.requirements.statistics.len() == 1)); } + /// An avg of per-series quantiles averages the quantiles at query time; + /// sum and count items would ask for sums of quantiles. + #[test] + fn per_series_avg_is_its_per_series_leaf() { + let query = "avg by (job) (quantile_over_time(0.99, metric[5m]))"; + assert_eq!( + decompose_to_leaves(query), + vec!["quantile_over_time(0.99, metric[5m])"] + ); + let items = extract_aqes(&[rqe(query, 60_000)], &empty_schema(), 15_000).unwrap(); + let [item] = items.as_slice() else { + panic!("{items:?}"); + }; + assert_eq!(item.requirements.statistics, vec![Statistic::Quantile]); + assert!(item.requirements.grouping_labels.labels.is_empty()); + } + #[test] fn spatial_only_query_sets_range_to_scrape_interval() { let rqes = vec![rqe("sum(metric)", 60_000)]; diff --git a/asap-planner-rs/src/optimizer/milp_output.rs b/asap-planner-rs/src/optimizer/milp_output.rs index 5e33ce98..49de8c19 100644 --- a/asap-planner-rs/src/optimizer/milp_output.rs +++ b/asap-planner-rs/src/optimizer/milp_output.rs @@ -24,13 +24,16 @@ use crate::planner_output::PlannerOutput; use crate::promql::generator::{build_inference_yaml, build_streaming_yaml}; use crate::sql::generator::{build_sql_inference_yaml, build_sql_streaming_yaml}; -use super::aqe_extractor::{contains_avg, decompose_to_leaves}; +use super::aqe_extractor::{contains_avg, decompose_to_leaves, per_series_avg}; use super::milp::{MilpConfig, MilpWorkload}; use super::solution::OptimizerItem; #[derive(Debug, Error)] pub enum MilpOutputError { - #[error("query {0:?} uses avg, which the engine can't answer from sum and count yet")] + #[error( + "query {0:?} uses an avg the engine can't answer yet: from sum and count, or in a \ + binary arm" + )] AvgQuery(String), #[error( "query groups set step_ms or range_duration_ms, whose extra retention MILP \ @@ -227,6 +230,7 @@ pub fn deployments_to_planner_output( } } } + plan_per_series_avgs(config, &binary_leaves, &mut queries)?; let id_map: HashMap = aggregations .keys() @@ -342,6 +346,50 @@ fn query_plan_entry( }) } +/// Plans each per-series avg query (`per_series_avg`) as its leaf's entry +/// plus a query-time avg, replacing the leaf's entry unless the leaf is +/// itself a query or a binary query's arm. +fn plan_per_series_avgs( + config: MilpConfig, + binary_leaves: &HashSet, + queries: &mut IndexMap, +) -> Result<(), MilpOutputError> { + let MilpConfig::Promql(config) = config else { + return Ok(()); + }; + let asked: Vec<&String> = config + .query_groups + .iter() + .flat_map(|group| &group.queries) + .collect(); + let mut leaves = Vec::new(); + for &query in &asked { + let Some((leaf, avg)) = per_series_avg(query) else { + continue; + }; + let unregroupable = || MilpOutputError::UnregroupableQuery(query.clone()); + let entry = queries.get(&leaf).ok_or_else(unregroupable)?; + // The leaf keeps every label, so its aggregation is never regrouped. + if !entry.query_time_aggregations.is_empty() { + return Err(unregroupable()); + } + let stage = aggregation_stage(&avg).ok_or_else(unregroupable)?; + let planned = QueryPlanEntry { + aggregation_keys: entry.aggregation_keys.clone(), + planned_subquery: leaf.clone(), + query_time_aggregations: vec![stage], + }; + queries.insert(query.clone(), planned); + leaves.push(leaf); + } + for leaf in leaves { + if !asked.contains(&&leaf) && !binary_leaves.contains(&leaf) { + queries.shift_remove(&leaf); + } + } + Ok(()) +} + /// The leaves of the workload's binary queries. The engine runs a binary /// query's arms without query-time stages. fn binary_query_leaves(config: MilpConfig) -> HashSet { @@ -1091,6 +1139,45 @@ mod tests { assert!(matches!(err, MilpOutputError::AvgQuery(q) if q == query)); } + /// An avg of a per-series function reads that function's aggregation + /// and averages its series at query time; the bare leaf isn't a query. + #[test] + fn per_series_avg_averages_its_leaf_at_query_time() { + let query = "avg by (job) (quantile_over_time(0.99, http_requests_total[5m]))"; + let leaf = "quantile_over_time(0.99, http_requests_total[5m])"; + let (config, workload, solution) = plan(&group(query, 0.99)); + let output = + plan_to_planner_output(MilpConfig::Promql(&config), &workload, &solution).unwrap(); + + let a = aggregation_for(&output, query); + assert_eq!(a.aggregation_type, AggregationType::DatasketchesKLL); + let query_config = query_config_for(&output, query); + assert_eq!(query_config.planned_subquery, leaf); + let [stage] = query_config.query_time_aggregations.as_slice() else { + panic!("{:?}", query_config.query_time_aggregations); + }; + assert_eq!( + serde_json::to_value(stage).unwrap(), + json!({"operator": "avg", "grouping": {"mode": "by", "labels": ["job"]}, + "parameter": null}) + ); + stage.validate().unwrap(); + let inference = output.to_inference_config(QueryLanguage::promql).unwrap(); + assert!(inference.query_configs.iter().all(|q| q.query != leaf)); + } + + /// A binary query's arms run without query-time stages, so a per-series + /// avg arm can't be answered. + #[test] + fn per_series_avg_in_a_binary_arm_is_an_error() { + let query = "avg by (job) (quantile_over_time(0.99, http_requests_total[5m])) * 2"; + let (config, workload, solution) = plan(&group(query, 0.99)); + let err = plan_to_planner_output(MilpConfig::Promql(&config), &workload, &solution) + .err() + .unwrap(); + assert!(matches!(err, MilpOutputError::AvgQuery(q) if q == query)); + } + /// Range overrides only size retention, so they block writing configs /// but not planning. #[test] diff --git a/asap-planner-rs/src/optimizer/saved_plan.rs b/asap-planner-rs/src/optimizer/saved_plan.rs index 325274ad..13588e62 100644 --- a/asap-planner-rs/src/optimizer/saved_plan.rs +++ b/asap-planner-rs/src/optimizer/saved_plan.rs @@ -108,7 +108,7 @@ struct Chosen { /// Index of the plan's deployment. deployment: usize, /// Present only when the deployment's grouping differs from the RQE's - /// (a roll-up), which the converter refuses. + /// (a roll-up). deployment_grouping: Option, } @@ -583,6 +583,41 @@ mod tests { ); } + /// An avg of per-series quantiles is one leaf, the per-series quantile + /// its one RQE names, averaged by `job` at query time. + #[test] + fn writes_a_per_series_avg_as_a_query_time_avg() { + let query = "avg by (job) (quantile_over_time(0.5, data[5m]))"; + let table = TABLE.replace( + r#""quantile_over_time(0.5, data[5m])", "interval_secs": 60"#, + &format!(r#""{query}", "interval_secs": 60"#), + ); + let inference = run_on(&table, plans(shared_p50()).to_string().as_bytes(), None) + .unwrap() + .to_inference_config(asap_types::enums::QueryLanguage::promql) + .unwrap(); + let query_config = inference + .query_configs + .iter() + .find(|q| q.query == query) + .expect("a query_config for the avg"); + assert_eq!( + query_config.planned_subquery, + "quantile_over_time(0.5, data[5m])" + ); + let [stage] = query_config.query_time_aggregations.as_slice() else { + panic!("{:?}", query_config.query_time_aggregations); + }; + assert_eq!( + serde_json::to_value(stage).unwrap(), + json!({ + "operator": "avg", + "grouping": {"mode": "by", "labels": ["job"]}, + "parameter": null, + }) + ); + } + #[test] fn sla_ms_selects_the_plan_at_that_sla() { let yaml = run(plans(shared_p50()), Some(100.0)) diff --git a/asap-query-engine/tests/e2e_precompute_equivalence.rs b/asap-query-engine/tests/e2e_precompute_equivalence.rs index 7ca83075..b0a1f3b0 100644 --- a/asap-query-engine/tests/e2e_precompute_equivalence.rs +++ b/asap-query-engine/tests/e2e_precompute_equivalence.rs @@ -2024,6 +2024,98 @@ async fn e2e_grouped_quantile_preserves_output_label_shape() { ); } +/// The average of per-series quantiles: per-series KLL windows merged over +/// the range, then averaged by group at query time. q=1 makes each series' +/// quantile its exact max, so a leaked or missing window changes the mean. +#[tokio::test] +async fn e2e_avg_of_per_series_quantile_over_time() { + use asap_types::query_config::{ + QueryTimeAggregationOperator, QueryTimeGrouping, QueryTimeGroupingMode, + }; + + let metric = "series_latency"; + let anchor = "quantile_over_time(1.0, series_latency[2s])"; + let query = "avg by (job) (quantile_over_time(1.0, series_latency[2s]))"; + let mut config = make_agg_config( + 19, + metric, + AggregationType::DatasketchesKLL, + "", + 1_000, + 0, + vec!["instance", "job"], + ); + config.parameters.insert("K".to_string(), json!(200_u64)); + // (instance, job, [(ts_ms, value)]): the 1_000 sample is outside (1s, 3s]. + let series = [ + ("a", "api", vec![(1_000, 100.0), (1_500, 1.0), (2_500, 9.0)]), + ("b", "api", vec![(1_500, 6.0), (2_500, 2.0)]), + ("c", "worker", vec![(1_500, 30.0), (2_500, 10.0)]), + ]; + let samples = series + .iter() + .flat_map(|(instance, job, points)| { + points + .iter() + .chain([(4_500, 0.0)].iter()) + .map(|&(ts, value)| { + make_timeseries( + metric, + vec![("instance", instance), ("job", job)], + ts, + value, + ) + }) + .collect::>() + }) + .collect(); + let (engine, query) = NativeDagScenario { + port: 19429, + metric, + query, + aggregation_configs: vec![config], + schema_labels: vec!["instance".to_string(), "job".to_string()], + samples, + evaluation_time_seconds: 3.0, + base_interval_ms: 1_000, + } + .build_engine_with_plan( + anchor, + vec![QueryTimeAggregation { + operator: QueryTimeAggregationOperator::Avg, + grouping: QueryTimeGrouping { + mode: QueryTimeGroupingMode::By, + labels: vec!["job".to_string()], + }, + parameter: None, + }], + ) + .await; + + let (output_labels, result) = engine + .handle_query_promql(query, 3.0) + .expect("avg of per-series quantiles should execute") + .expect("avg of per-series quantiles should match configured inference"); + assert_eq!(output_labels.labels, vec!["job"]); + let QueryResult::Vector(vector) = result else { + panic!("expected instant vector result"); + }; + let mut values: Vec<_> = vector + .values + .into_iter() + .map(|element| (element.labels.labels, element.value)) + .collect(); + values.sort_by(|a, b| a.0.cmp(&b.0)); + // api: mean of max(1, 9) and max(6, 2); worker: max(30, 10). + assert_eq!( + values, + vec![ + (vec!["api".to_string()], 7.5), + (vec!["worker".to_string()], 30.0) + ] + ); +} + /// A grouped quantile served by a DDSketch aggregation: samples from two /// instances per job flow through remote write and precompute into one /// DDSketch per group, and each returned quantile stays within the sketch's diff --git a/asap-tools/data-sources/paced-feeder/README.md b/asap-tools/data-sources/paced-feeder/README.md index ab41bd70..39d23e10 100644 --- a/asap-tools/data-sources/paced-feeder/README.md +++ b/asap-tools/data-sources/paced-feeder/README.md @@ -97,6 +97,11 @@ name), `n` and the draw, so records match sketch-bench's `label_shares()` `endpoint="3.17"`). - **Keys** (`zipf` θ over `keys`): the key of Zipf rank `r` is `r`, so key 0 is the most frequent. +- **Rotating per-series pool** (`{"rotating_pool": {"universe_keys": U, + "keys_per_series": K}}`): records enumerate every label combination evenly; + each combination cycles through a deterministic K-key pool in the U-key + universe. Pool starts are seeded per combination, so pools overlap while + each leaf has exactly K possible keys. - **Burst**: with probability `share` a record is a burst record: its `label` is `value` (a child index for a child label) and its key is uniform over `keys` burst keys numbered after the background keys (`1000000` to diff --git a/asap-tools/data-sources/paced-feeder/src/records.rs b/asap-tools/data-sources/paced-feeder/src/records.rs index 7075d17b..2e15c9ef 100644 --- a/asap-tools/data-sources/paced-feeder/src/records.rs +++ b/asap-tools/data-sources/paced-feeder/src/records.rs @@ -74,11 +74,19 @@ struct FieldSpec { zipf: Option, keys: Option, burst: Option, + rotating_pool: Option, pareto: Option, scaled_by: Option, anomaly: Option, } +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RotatingPoolSpec { + universe_keys: u64, + keys_per_series: u64, +} + #[derive(Deserialize)] #[serde(deny_unknown_fields)] struct BurstSpec { @@ -169,6 +177,10 @@ enum Field { zipf: Zipf, keys: u64, }, + RotatingPool { + universe_keys: u64, + keys_per_series: u64, + }, Pareto { shape: f64, /// The `scaled_by` label and each of its values' factor. @@ -190,6 +202,8 @@ pub struct RecordMetric { /// Value field names, in name order. pub fields: Vec, field_kinds: Vec, + round_robin_series: bool, + series_count: u64, pub burst: Option, pub records_per_sec: u64, seed: u64, @@ -258,6 +272,7 @@ impl RecordMetric { zipf: Some(theta), keys: Some(keys), burst: field_burst, + rotating_pool: None, pareto: None, scaled_by: None, anomaly: None, @@ -301,6 +316,33 @@ impl RecordMetric { zipf: None, keys: None, burst: None, + rotating_pool: Some(pool), + pareto: None, + scaled_by: None, + anomaly: None, + } => { + ensure!( + pool.universe_keys > 0 && pool.keys_per_series > 0, + "value `{field}`: rotating pool sizes must be positive" + ); + ensure!( + pool.keys_per_series <= pool.universe_keys, + "value `{field}`: rotating pool must fit in its universe" + ); + ensure!( + pool.universe_keys <= MAX_KEY, + "value `{field}`: rotating pool keys must be ≤ 2^53" + ); + Field::RotatingPool { + universe_keys: pool.universe_keys, + keys_per_series: pool.keys_per_series, + } + } + FieldSpec { + zipf: None, + keys: None, + burst: None, + rotating_pool: None, pareto: Some(shape), scaled_by, anomaly, @@ -346,6 +388,19 @@ impl RecordMetric { } let records_per_sec = records_per_sec.unwrap_or(spec.records_per_sec); ensure!(records_per_sec > 0, "records per second must be positive"); + let series_count = labels + .iter() + .try_fold(1u64, |count, label| count.checked_mul(label.cardinality)) + .context("too many record label combinations")?; + let round_robin_series = field_kinds + .iter() + .any(|field| matches!(field, Field::RotatingPool { .. })); + if round_robin_series { + ensure!( + records_per_sec % series_count == 0, + "records per second must be a multiple of record label combinations for rotating pools" + ); + } Ok(Self { name: name.to_string(), labels, @@ -354,6 +409,8 @@ impl RecordMetric { label_zipfs, fields, field_kinds, + round_robin_series, + series_count, burst, records_per_sec, seed: splitmix64(seed ^ fnv1a(name)), @@ -369,15 +426,33 @@ impl RecordMetric { let stream = num_labels + self.fields.len() as u64; u(stream) <= b.share }); - for (i, zipf) in self.label_zipfs.iter().enumerate() { - let child = match burst { - Some(b) if b.label == i => b.value, - _ => zipf.draw(u(i as u64)), - }; - labels[i] = match self.parents[i] { - None => child, - Some(p) => u64::from(labels[p]) * self.labels[i].cardinality + child, - } as u32; + if self.round_robin_series { + let mut series = index % self.series_count; + let mut children = vec![0u64; self.labels.len()]; + for i in (0..self.labels.len()).rev() { + let cardinality = self.labels[i].cardinality; + children[i] = series % cardinality; + series /= cardinality; + } + for i in 0..self.labels.len() { + let cardinality = self.labels[i].cardinality; + let child = children[i]; + labels[i] = match self.parents[i] { + None => child, + Some(p) => u64::from(labels[p]) * cardinality + child, + } as u32; + } + } else { + for (i, zipf) in self.label_zipfs.iter().enumerate() { + let child = match burst { + Some(b) if b.label == i => b.value, + _ => zipf.draw(u(i as u64)), + }; + labels[i] = match self.parents[i] { + None => child, + Some(p) => u64::from(labels[p]) * self.labels[i].cardinality + child, + } as u32; + } } for (j, kind) in self.field_kinds.iter().enumerate() { let u = u(num_labels + j as u64); @@ -389,6 +464,15 @@ impl RecordMetric { } _ => zipf.draw(u) as f64, }, + Field::RotatingPool { + universe_keys, + keys_per_series, + } => { + let series = index % self.series_count; + let pool_start = splitmix64(self.seed ^ series) % universe_keys; + let rotation = (index / self.series_count) % keys_per_series; + ((pool_start + rotation) % universe_keys) as f64 + } Field::Pareto { shape, scaled_by, @@ -637,6 +721,38 @@ pub mod tests { RecordMetric::from_table(name, spec, None, 42).unwrap() } + #[test] + fn rotating_pool_cycles_each_leafs_overlapping_key_pool() { + let spec = json!({ + "type": "records", "records_per_sec": 12, + "labels": [ + {"name": "region", "cardinality": 2, "skew": 0.0}, + {"name": "leaf", "cardinality": 3, "child_of": "region", "skew": 0.0} + ], + "values": {"user_id": {"rotating_pool": { + "universe_keys": 10, "keys_per_series": 4 + }}} + }); + let metric = metric("events", &spec); + let rows = records(&metric, 24); + + for (index, (labels, _)) in rows.iter().enumerate() { + let series = index as u64 % 6; + assert_eq!(labels[0], (series / 3) as u32); + assert_eq!(labels[1], series as u32); + } + for series in 0..6 { + let values: HashSet = rows + .iter() + .skip(series) + .step_by(6) + .map(|(_, values)| values[0] as u64) + .collect(); + assert_eq!(values.len(), 4); + assert!(values.iter().all(|&value| value < 10)); + } + } + /// Records `0..n` as (label value indices, values). fn records(m: &RecordMetric, n: u64) -> Vec<(Vec, Vec)> { (0..n) diff --git a/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py b/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py index 96690cf9..1ec9b934 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py +++ b/asap-tools/experiments/datasets/fake_metric/random_workload_artifacts.py @@ -1,15 +1,39 @@ #!/usr/bin/env python3 -"""Render frozen random-workload manifests into offline planner inputs.""" +"""Render frozen random-workload manifests into offline planner inputs and, +with --pilot-dir, the five-system pilot's run configs.""" import argparse +import copy import json +import os +import sys from collections import defaultdict from pathlib import Path import yaml +import sketch_bench_workloads as sbw +from random_workloads import CLICKHOUSE_LABELS, rqe_query + +EXPERIMENTS_DIR = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +) +sys.path.insert(0, EXPERIMENTS_DIR) +from sketch_bench_e2e.sketch_bench_e2e_configs import ( # noqa: E402 + BASELINE_MODE, + build_config, +) ACCURACY_SLA = 0.95 +PILOT_SUPPORT = "common-promql" +PILOT_DURATION_MS = 10 * 60 * 1000 +QUERY_FRESHNESS_DELAY_SECS = 10 +# Relative to EXPERIMENTS_DIR, where the runners read configs and SQL. +CONFIG_DIR = "config/experiment_type/random_workload" +VICTORIAMETRICS_URL = "http://localhost:8428" +# 26.6 and later need AVX2, which CloudLab's Xeon E5-2450 nodes lack; the +# ClickHouse SQL's semantics tests also pass on 23.8. +CLICKHOUSE_IMAGE_TAG = "23.8" PROMQL_METRICS = ("data", "data_total") SQL_EVENTS_TABLE = { "name": "events", @@ -19,6 +43,66 @@ } +def paced_feeder_table(workload): + """Render the manifest's data facts as a paced-feeder workload table.""" + facts = workload["data_facts"] + labels = workload["labels"] + series = facts["series"] + samples_per_sec = workload["samples_per_sec_per_series"] + return { + "workloads": [ + { + "e2e": { + "metrics": { + "data": { + "type": "gauge", + "labels": labels, + "samples_per_sec_per_series": samples_per_sec, + "values": { + "value": { + "zipf": series["ranking"]["exponent"], + "keys": series["leaf_count"], + }, + "dist": { + "pareto": facts["metrics"]["data"]["pareto_shape"] + }, + }, + }, + "data_total": { + "type": "counter", + "labels": labels, + "samples_per_sec_per_series": samples_per_sec, + "values": { + "increment": { + "zipf": series["ranking"]["exponent"], + "keys": series["leaf_count"], + } + }, + }, + "events": { + "type": "records", + "records_per_sec": series["leaf_count"] * samples_per_sec, + "labels": [{**label, "skew": 0.0} for label in labels], + "values": { + facts["events"]["distinct_key"]: { + "rotating_pool": { + "universe_keys": facts["events"][ + "user_id_universe_size" + ], + "keys_per_series": facts["events"][ + "user_ids_per_leaf" + ], + } + } + }, + }, + } + } + } + ] + } + + def query_groups(queries, query_key): """Group rendered query text by its fixed repetition interval.""" grouped = defaultdict(list) @@ -39,12 +123,123 @@ def query_groups(queries, query_key): ] +def pilot_configs(workload, pilot_dir): + """The five-system pilot's run configs, {path under EXPERIMENTS_DIR: + content}: ASAPQuery (the saved plan in `pilot_dir`) and Prometheus in + one experiment_run_e2e.py config, VictoriaMetrics in another, and raw + and MV ClickHouse in one experiment_run_clickhouse.py config each. + + Every arm feeds the metrics of `pilot_dir`'s eval table with the + manifest's feeder seed, waits out the longest lookback plus the + freshness delay, then evaluates each query at multiples of its interval + for PILOT_DURATION_MS: the same windows relative to its data start. + Raw ClickHouse runs the manifest's clickhouse_sql, and ClickHouse+MV + the same RQEs' Query over exact views (sketch_bench_workloads).""" + queries = [q for q in workload["queries"] if q["support"] == PILOT_SUPPORT] + lookback_secs = max(q["lookback_ms"] for q in queries) // 1000 + starting_delay = lookback_secs + QUERY_FRESHNESS_DELAY_SECS + duration = PILOT_DURATION_MS // 1000 + seed = workload["data_facts"]["feeder_seed"] + table = pilot_dir / "eval_table.json" + out_dir = Path(CONFIG_DIR) / workload["id"] + + e2e, _ = build_config( + str(table), + seed, + starting_delay, + duration, + QUERY_FRESHNESS_DELAY_SECS, + prebuilt_output_dir=str(pilot_dir), + ) + # One monitoring tool per run, so VictoriaMetrics is its own baseline run. + vm = copy.deepcopy(e2e) + del vm["controller"] + vm_params = vm["experiment_params"] + vm_params["experiment"] = [BASELINE_MODE] + vm_params["monitoring"] = { + "tool": "victoriametrics", + "deployment_mode": "containerized", + } + for server in vm_params["servers"]: + if server["name"] == "prometheus": + server["url"] = VICTORIAMETRICS_URL + + metrics = json.loads(table.read_text())["workloads"][0]["e2e"]["metrics"] + rqe_queries = [rqe_query(query["rqes"][0]) for query in queries] + views = sbw.plan_views(rqe_queries, metrics) + files = { + "init.sql": sbw.init_sql(metrics, [], create_views=False), + "init_mv.sql": sbw.init_sql(metrics, list(views.values()), create_views=True), + } + statements = defaultdict(lambda: {sbw.BASELINE: [], sbw.BASELINE_MV: []}) + for query, q in zip(queries, rqe_queries): + sql = { + sbw.BASELINE: query["clickhouse_sql"], + sbw.BASELINE_MV: sbw.mv_sql(q, views, CLICKHOUSE_LABELS), + } + for mode, text in sql.items(): + statements[q.interval_secs][mode].append(f"-- {query['promql']}\n{text}") + groups = [] + for interval, by_mode in sorted(statements.items()): + sql_files = {} + for mode, texts in by_mode.items(): + name = f"interval_{interval}s_{mode}.sql" + files[name] = sbw.statements_text(texts) + sql_files[mode] = str(out_dir / name) + groups.append( + { + "name": f"interval_{interval}s", + "sql_file": sql_files, + "repetition_delay_ms": interval * 1000, + # The SQL reads only {{ end_time }}; the client requires a value. + "time_window_seconds": 0, + "client_options": { + "starting_delay": starting_delay, + "repetitions": duration // interval, + "align_to_interval": True, + "query_time_offset": QUERY_FRESHNESS_DELAY_SECS, + }, + } + ) + feeder = { + "workload_table": str(table.resolve()), + "metrics": list(metrics), + "seed": seed, + "duration_secs": starting_delay + duration + sbw.FEEDER_SLACK_SECS, + } + for mode, init, name in ( + (sbw.BASELINE, "init.sql", "clickhouse.yaml"), + (sbw.BASELINE_MV, "init_mv.sql", "clickhouse_mv.yaml"), + ): + clickhouse = { + "experiment": [{"mode": mode, "server": "clickhouse"}], + "servers": [{"name": "clickhouse", "url": sbw.CLICKHOUSE_URL}], + "dataset": { + "name": "custom", + "init_sql_file": str(out_dir / init), + "paced_feeder": feeder, + "clickhouse_image_tag": CLICKHOUSE_IMAGE_TAG, + }, + "query_groups": groups, + "data_ingestion_interval_ms": workload["data_facts"]["scrape_interval_ms"], + } + files[name] = "# @package experiment_params\n" + yaml.safe_dump( + clickhouse, sort_keys=False + ) + for name, config in (("e2e.yaml", e2e), ("victoriametrics.yaml", vm)): + files[name] = "# @package _global_\n" + yaml.safe_dump(config, sort_keys=False) + return {out_dir / name: content for name, content in files.items()} + + def render(workload): """Return planner inputs and query schedule derived from one manifest.""" labels = [label["name"] for label in workload["labels"]] queries = workload["queries"] return { + "workload_id": workload["id"], + "data_facts": workload["data_facts"], "queries": queries, + "paced_feeder_table": paced_feeder_table(workload), "promql_planner": { "metrics": [ {"metric": metric, "labels": labels} for metric in PROMQL_METRICS @@ -64,6 +259,12 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, required=True) parser.add_argument("--out", type=Path, required=True) + parser.add_argument( + "--pilot-dir", + type=Path, + help="the workload's archived pilot plan (eval_table.json and planner " + f"configs): also write the pilot's run configs under {CONFIG_DIR}", + ) args = parser.parse_args() workload = json.loads(args.manifest.read_text()) rendered = render(workload) @@ -74,9 +275,29 @@ def main(): print(path) schedule = args.out / "queries.json" schedule.write_text( - json.dumps(rendered["queries"], indent=2, sort_keys=True) + "\n" + json.dumps( + { + "workload_id": rendered["workload_id"], + "data_facts": rendered["data_facts"], + "queries": rendered["queries"], + }, + indent=2, + sort_keys=True, + ) + + "\n" ) print(schedule) + table = args.out / "paced_feeder_table.json" + table.write_text( + json.dumps(rendered["paced_feeder_table"], indent=2, sort_keys=True) + "\n" + ) + print(table) + if args.pilot_dir: + for path, content in pilot_configs(workload, args.pilot_dir).items(): + path = Path(EXPERIMENTS_DIR) / path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + print(path) if __name__ == "__main__": diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads.py b/asap-tools/experiments/datasets/fake_metric/random_workloads.py index 7d030780..9e39a690 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads.py +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads.py @@ -12,6 +12,7 @@ import random from pathlib import Path +from sketch_bench_workloads import Query, raw_sql WORKLOAD_COUNT = 10 QUERIES_PER_WORKLOAD = 10 @@ -19,8 +20,10 @@ ACCURACY_SLA = 0.95 SECOND_MS = 1_000 MINUTE_MS = 60 * SECOND_MS -SPATIAL_LOOKBACK_MS = 100 -SPATIAL_INTERVAL_MS = 100 +# 100 s at 10 samples/s gives each series' quantile sketch 1,000 samples, the +# first N sketch-bench's accuracy curves cover. +QUANTILE_LOOKBACK_MS = 100 * SECOND_MS +QUANTILE_INTERVAL_MS = SECOND_MS LABEL_HIERARCHY = ( ("region", 4, None), ("service", 5, "region"), @@ -49,6 +52,16 @@ ) QUANTILES = (0.50, 0.75, 0.90, 0.95, 0.99) TOPK_VALUES = (4, 8) +BACKFILL_HISTORY_MS = 20 * MINUTE_MS +LEAF_SERIES_COUNT = math.prod(LABEL_CARDINALITIES.values()) +LEAF_LABELS = tuple(name for name, _, _ in LABEL_HIERARCHY) +# Every ClickHouse table has the leaf labels (sketch_bench_workloads' tables). +CLICKHOUSE_LABELS = { + table: list(LEAF_LABELS) for table in ("data", "data_total", "events") +} +ZIPF_EXPONENT = 1.1 +USER_ID_UNIVERSE_SIZE = 10_000 +USER_IDS_PER_LEAF = 1_000 def duration(milliseconds): @@ -60,11 +73,100 @@ def grouping_text(grouping): return ", ".join(grouping) +def rqe_query(rqe): + """The ClickHouse renderers' Query for one RQE, built from its fields, + never from its PromQL. Raw and MV ClickHouse SQL both render it.""" + capability, grouping = rqe["capability"], tuple(rqe["grouping_labels"]) + common = { + "table": rqe["metric"], + "interval_secs": rqe["interval_ms"] // SECOND_MS, + "window_secs": rqe["lookback_ms"] // SECOND_MS, + } + if capability == "sum": + return Query(**common, grouping=grouping, series_fn=None, agg="sum") + if capability == "quantile": + quantile = f"{rqe['phi']:.2f}" + return Query( + **common, + grouping=grouping, + series_fn="quantile", + agg="avg", + quantile=quantile, + ) + if capability == "topk_by_value": + return Query( + **common, + grouping=LEAF_LABELS, + series_fn=None, + agg="sum", + topk=rqe["k"], + topk_by=grouping, + ) + if capability in ("topk_by_count", "rate"): + series_fn = "increase" if capability == "topk_by_count" else "rate" + return Query( + **common, + grouping=LEAF_LABELS, + series_fn=series_fn, + agg=None, + topk=rqe["k"], + topk_by=grouping, + ) + if capability == "cardinality": + # Planner SQL windows are left-open, (E - W, E]. + return Query( + **common, + grouping=grouping, + series_fn=None, + agg="cardinality", + value=rqe["value"], + left_open=True, + ) + raise ValueError(f"unknown capability {capability!r}") + + def choose_temporal_time(rng): lookback_ms, intervals = rng.choice(TEMPORAL_TIMES) return lookback_ms, rng.choice(intervals) +def workload_data_facts(seed): + """Return the feeder contract shared by every workload from one seed.""" + return { + "feeder_seed": seed, + "scrape_interval_ms": SECOND_MS // SAMPLES_PER_SEC_PER_SERIES, + "backfill_history_ms": BACKFILL_HISTORY_MS, + "series": { + "leaf_count": LEAF_SERIES_COUNT, + "ranking": { + "kind": "deterministic_zipf_like", + "exponent": ZIPF_EXPONENT, + "rank_scope": "leaf_series", + }, + }, + "metrics": { + "data": { + "kind": "gauge", + "value_shape": "rank_scaled_pareto", + "pareto_shape": 2.0, + }, + "data_total": { + "kind": "counter", + "increment_shape": "rank_scaled_positive_deterministic", + "resets": "none", + }, + }, + "events": { + "table": "events", + "time_column": "ts", + "distinct_key": "user_id", + "user_id_universe_size": USER_ID_UNIVERSE_SIZE, + "user_ids_per_leaf": USER_IDS_PER_LEAF, + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + }, + } + + def query( query_id, capability, @@ -76,10 +178,15 @@ def query( **params, ): """Create one manifest query and its single optimizer-facing RQE.""" + metrics = { + "rate": "data_total", + "topk_by_count": "data_total", + "cardinality": "events", + } rqe = { "id": f"{query_id}/rqe", "capability": capability, - "metric": "data_total" if capability in ("rate", "topk_by_count") else "data", + "metric": metrics.get(capability, "data"), "grouping_labels": list(grouping), "lookback_ms": lookback_ms, "interval_ms": interval_ms, @@ -96,6 +203,7 @@ def query( "interval_ms": interval_ms, "promql": rendered if support != "sql-only" else None, "sql": rendered if support == "sql-only" else None, + "clickhouse_sql": raw_sql(rqe_query(rqe), CLICKHOUSE_LABELS), "rqes": [rqe], } @@ -103,18 +211,24 @@ def query( def sum_query(query_id, grouping, lookback_ms, interval_ms): rendered = f"sum by ({grouping_text(grouping)}) (sum_over_time(data[{duration(lookback_ms)}]))" return query( - query_id, "sum", grouping, lookback_ms, interval_ms, rendered, "common-promql" + query_id, + "sum", + grouping, + lookback_ms, + interval_ms, + rendered, + "common-promql", ) def quantile_query(query_id, grouping, phi): - rendered = f"quantile by ({grouping_text(grouping)}) ({phi:.2f}, data)" + rendered = f"avg by ({grouping_text(grouping)}) (quantile_over_time({phi:.2f}, data[{QUANTILE_LOOKBACK_MS // SECOND_MS}s]))" return query( query_id, "quantile", grouping, - SPATIAL_LOOKBACK_MS, - SPATIAL_INTERVAL_MS, + QUANTILE_LOOKBACK_MS, + QUANTILE_INTERVAL_MS, rendered, "common-promql", phi=phi, @@ -235,6 +349,7 @@ def generate_workload(seed, index): "seed": seed, "labels": METRIC_LABELS, "samples_per_sec_per_series": SAMPLES_PER_SEC_PER_SERIES, + "data_facts": workload_data_facts(seed), "queries": queries, } validate_workload(workload) @@ -244,6 +359,16 @@ def generate_workload(seed, index): def validate_workload(workload): """Reject manifests that violate the experiment's stable public contract.""" queries = workload["queries"] + data_facts = workload["data_facts"] + if data_facts["scrape_interval_ms"] != SECOND_MS // SAMPLES_PER_SEC_PER_SERIES: + raise ValueError("data facts must declare the 100 ms scrape interval") + if data_facts["series"]["leaf_count"] != LEAF_SERIES_COUNT: + raise ValueError("data facts must declare every leaf series") + if data_facts["metrics"]["data_total"]["resets"] != "none": + raise ValueError("the pilot counter contract must be monotonic") + events = data_facts["events"] + if not 0 < events["user_ids_per_leaf"] <= events["user_id_universe_size"]: + raise ValueError("distinct user pools must fit in the declared universe") if len(queries) != QUERIES_PER_WORKLOAD: raise ValueError( f"a workload must contain exactly {QUERIES_PER_WORKLOAD} queries" diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/README.md b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/README.md new file mode 100644 index 00000000..321063f2 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/README.md @@ -0,0 +1,38 @@ +# Workload 01 pilot plan + +The saved sketch-bench plan for workload 01's seven common-promql queries at +100 ms ingestion, and its ASAPQuery configs: + +- `eval_table.json`: sketch-bench's eval table of the manifest + (`export_autosketch_eval_table.py --random-manifest`). +- `plans.json.gz`: each method's chosen plans (`autosketch_vs_asap`, p95). +- `streaming_config.yaml`, `inference_config.yaml`: the cheapest `asap` plan + on CPU weights, through `asap-planner --planner saved`. + +The `asap` plan serves the coarse sum from the fine sum's deployment (a +roll-up) and the three quantiles from one per-series DDSketch. Top-k heaps +over 500 series read the curves at their first 1,000 keys (the table's +`assumptions`). + +`../workload-NN-pilot/` hold workloads 02-10's plans, made the same way +(sketch-bench f885d7e: queries reading the same RQE share it). + +Regenerate, with `SB` a sketch-bench checkout at the table's +`sketch_bench_revision`, from this repository's root: + +```bash +DIR=asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862 +OUT=$DIR/workload-01-pilot; SAT=$(mktemp -d); TMP=$(mktemp -d) +cp -r $SB/rqe-optimizer/results/autosketch-vs-asap-inputs/saturation/out_* $SAT/ +cp -r $SB/rqe-optimizer/results/autosketch-vs-asap-hydra/inputs/saturation/* $SAT/ +(cd $SB && cargo build --release -p rqe-optimizer --examples) +python3 $SB/scripts/export_autosketch_eval_table.py \ + --random-manifest $DIR/workload-01.json --out $OUT/eval_table.json +$SB/target/release/examples/autosketch_vs_asap synthetic --table $OUT/eval_table.json \ + --target p95 --runs 1 --saturation-dir $SAT --out $TMP/tp95.json +python3 $SB/scripts/extract_autosketch_plans.py $TMP/tp95.json $OUT/plans.json.gz +cargo run --release --manifest-path asap-planner-rs/Cargo.toml --bin asap-planner -- \ + --planner saved --workload-table $OUT/eval_table.json --plan $OUT/plans.json.gz \ + --method asap --weights cpu --output_dir $OUT \ + --streaming_engine precompute --data-ingestion-interval-ms 100 +``` diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/eval_table.json new file mode 100644 index 00000000..38e91a55 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/eval_table.json @@ -0,0 +1,448 @@ +{ + "inputs": { + "workload-01.json": "edacdc8662d75c0cd1c8bcc3136614225fd9f273d98360e73d8c563016586ae4" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "85699c9edef79ed6362ff19020d4c1f486af531d", + "workloads": [ + { + "dataset": "random/seed=862/workload-01", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 10000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "interval_secs": 20, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-01/q01-sum-fine/keys/600s" + ] + }, + { + "interval_secs": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-01/q02-sum-coarse/keys/600s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q04-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q05-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-01/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "interval_secs": 30, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-01/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-01/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 300000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/inference_config.yaml new file mode 100644 index 00000000..9a005085 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/inference_config.yaml @@ -0,0 +1,80 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[10m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[10m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[10m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[10m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 59 + query: topk by (region, service) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 40 + query: topk by (region, service) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/plans.json.gz new file mode 100644 index 00000000..dcd2f883 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/streaming_config.yaml new file mode 100644 index 00000000..6391395a --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 25000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 20000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: sliding +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 1024 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01.json index 0e3f9250..90fe68c0 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01.json +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-01.json @@ -1,4 +1,37 @@ { + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 10000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, "id": "workload-01", "labels": [ { @@ -23,6 +56,7 @@ ], "queries": [ { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -53,6 +87,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service", "grouping": [ "region", "service" @@ -81,17 +116,18 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", "grouping": [ "region", "service", "job" ], "id": "q03-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 200, "output_series_upper_bound": 200, - "promql": "quantile by (region, service, job) (0.99, data)", + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -102,8 +138,8 @@ "job" ], "id": "q03-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.99 } @@ -112,15 +148,16 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", "grouping": [ "region" ], "id": "q04-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 4, "output_series_upper_bound": 4, - "promql": "quantile by (region) (0.95, data)", + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -129,8 +166,8 @@ "region" ], "id": "q04-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.95 } @@ -139,17 +176,18 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", "grouping": [ "region", "service", "job" ], "id": "q05-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 200, "output_series_upper_bound": 200, - "promql": "quantile by (region, service, job) (0.75, data)", + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -160,8 +198,8 @@ "job" ], "id": "q05-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.75 } @@ -170,6 +208,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", "grouping": [ "region", "service" @@ -199,6 +238,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", "grouping": [ "region", "service" @@ -228,6 +268,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 20) * 20 - 600) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 20) * 20 - 600)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 20) * 20 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 20) * 20 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", "grouping": [ "region", "service" @@ -257,6 +298,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -284,6 +326,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service", "grouping": [ "region", "service" @@ -305,7 +348,7 @@ "id": "q10-cardinality/rqe", "interval_ms": 20000, "lookback_ms": 600000, - "metric": "data", + "metric": "events", "value": "user_id" } ], diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02-pilot/eval_table.json new file mode 100644 index 00000000..24633c35 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02-pilot/eval_table.json @@ -0,0 +1,448 @@ +{ + "inputs": { + "workload-02.json": "0e310ace0829cf327f912ad3885a87555bcbadd742c73faa99224103b7542bd8" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "f885d7e9a99dd435683a7dcd66516a36b3ac02ab", + "workloads": [ + { + "dataset": "random/seed=862/workload-02", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 10000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "interval_secs": 5, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-02/q01-sum-fine/keys/300s" + ] + }, + { + "interval_secs": 5, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-02/q02-sum-coarse/keys/300s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-02/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-02/q04-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-02/q05-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-02/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "interval_secs": 60, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-02/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 150000.0, + "covered_min_N": 150000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-02/q01-sum-fine/keys/300s", + "instant": false, + "interval_secs": 5, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 150000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 150000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 1500000.0, + "covered_min_N": 1500000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-02/q02-sum-coarse/keys/300s", + "instant": false, + "interval_secs": 5, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 1500000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 1500000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-02/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-02/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 300000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-02", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-02/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02-pilot/inference_config.yaml new file mode 100644 index 00000000..3d590028 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02-pilot/inference_config.yaml @@ -0,0 +1,79 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service) (sum_over_time(data[5m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[5m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service, job) (sum_over_time(data[5m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 59 + query: topk by (region, service) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 20 + query: topk by (region, service) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02-pilot/plans.json.gz new file mode 100644 index 00000000..25193416 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02-pilot/streaming_config.yaml new file mode 100644 index 00000000..984fbe74 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 25000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 5000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 5000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: sliding +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 80 + width: 4096 + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02.json index 171e2d0a..8dfd95b9 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02.json +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-02.json @@ -1,4 +1,37 @@ { + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 10000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, "id": "workload-02", "labels": [ { @@ -23,6 +56,7 @@ ], "queries": [ { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -53,6 +87,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service", "grouping": [ "region", "service" @@ -81,15 +116,16 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", "grouping": [ "region" ], "id": "q03-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 4, "output_series_upper_bound": 4, - "promql": "quantile by (region) (0.75, data)", + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -98,8 +134,8 @@ "region" ], "id": "q03-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.75 } @@ -108,16 +144,17 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", "grouping": [ "region", "service" ], "id": "q04-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 20, "output_series_upper_bound": 20, - "promql": "quantile by (region, service) (0.95, data)", + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -127,8 +164,8 @@ "service" ], "id": "q04-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.95 } @@ -137,17 +174,18 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", "grouping": [ "region", "service", "job" ], "id": "q05-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 200, "output_series_upper_bound": 200, - "promql": "quantile by (region, service, job) (0.99, data)", + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -158,8 +196,8 @@ "job" ], "id": "q05-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.99 } @@ -168,6 +206,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", "grouping": [ "region", "service" @@ -197,6 +236,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", "grouping": [ "region", "service" @@ -226,6 +266,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 5) * 5 - 300) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 5) * 5 - 300)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 5) * 5 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 5) * 5 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -253,6 +294,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -280,6 +322,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -303,7 +346,7 @@ "id": "q10-cardinality/rqe", "interval_ms": 5000, "lookback_ms": 300000, - "metric": "data", + "metric": "events", "value": "user_id" } ], diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03-pilot/eval_table.json new file mode 100644 index 00000000..69535a59 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03-pilot/eval_table.json @@ -0,0 +1,392 @@ +{ + "inputs": { + "workload-03.json": "7f4825f0d6dfaecb7a17459b304a14fa0fa1f6155671b533c6068e03e495d4ff" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "f885d7e9a99dd435683a7dcd66516a36b3ac02ab", + "workloads": [ + { + "dataset": "random/seed=862/workload-03", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 10000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "interval_secs": 1, + "promql": "sum by (region, service, job) (sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-03/q01-sum-fine/keys/60s" + ] + }, + { + "interval_secs": 1, + "promql": "sum by (region, service) (sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-03/q02-sum-coarse/keys/60s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-03/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-03/q04-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-03/q04-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-03/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "interval_secs": 30, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-03/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 30000.0, + "covered_min_N": 30000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-03/q01-sum-fine/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 60.0, + "max_N": 30000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 30000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-03/q02-sum-coarse/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 60.0, + "max_N": 300000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-03/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 1500000.0, + "covered_min_N": 1500000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "topk", + "grid_K": 2500, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-03/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 4, + "keys_per_window": 2500 + }, + "lookback_secs": 60.0, + "max_N": 1500000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 1500000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 30000000.0, + "covered_min_N": 30000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-03", + "families": [ + { + "family": "topk", + "grid_K": 2500, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-03/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 4, + "keys_per_window": 2500 + }, + "lookback_secs": 1200.0, + "max_N": 30000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 30000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03-pilot/inference_config.yaml new file mode 100644 index 00000000..7a66e5b6 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03-pilot/inference_config.yaml @@ -0,0 +1,80 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service) (sum_over_time(data[1m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[1m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service, job) (sum_over_time(data[1m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 59 + query: topk by (region) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 40 + query: topk by (region) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service, job) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03-pilot/plans.json.gz new file mode 100644 index 00000000..d1ab64fd Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03-pilot/streaming_config.yaml new file mode 100644 index 00000000..665324ee --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 1000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: sliding +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03.json index 5d93cf32..d38c985a 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03.json +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-03.json @@ -1,4 +1,37 @@ { + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 10000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, "id": "workload-03", "labels": [ { @@ -23,6 +56,7 @@ ], "queries": [ { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -53,6 +87,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service", "grouping": [ "region", "service" @@ -81,16 +116,17 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", "grouping": [ "region", "service" ], "id": "q03-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 20, "output_series_upper_bound": 20, - "promql": "quantile by (region, service) (0.95, data)", + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -100,8 +136,8 @@ "service" ], "id": "q03-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.95 } @@ -110,16 +146,17 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", "grouping": [ "region", "service" ], "id": "q04-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 20, "output_series_upper_bound": 20, - "promql": "quantile by (region, service) (0.90, data)", + "promql": "avg by (region, service) (quantile_over_time(0.90, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -129,8 +166,8 @@ "service" ], "id": "q04-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.9 } @@ -139,17 +176,18 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", "grouping": [ "region", "service", "job" ], "id": "q05-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 200, "output_series_upper_bound": 200, - "promql": "quantile by (region, service, job) (0.90, data)", + "promql": "avg by (region, service, job) (quantile_over_time(0.90, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -160,8 +198,8 @@ "job" ], "id": "q05-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.9 } @@ -170,6 +208,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -197,6 +236,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -224,6 +264,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 1) * 1 - 60) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 1) * 1 - 60)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 1) * 1 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 1) * 1 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", "grouping": [ "region", "service" @@ -253,6 +294,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -280,6 +322,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -303,7 +346,7 @@ "id": "q10-cardinality/rqe", "interval_ms": 1000, "lookback_ms": 60000, - "metric": "data", + "metric": "events", "value": "user_id" } ], diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04-pilot/eval_table.json new file mode 100644 index 00000000..e543cde2 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04-pilot/eval_table.json @@ -0,0 +1,395 @@ +{ + "inputs": { + "workload-04.json": "4536c67e518848a19b023b12e4c117dc3006dc672f74eb327a9fa688391c7213" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "f885d7e9a99dd435683a7dcd66516a36b3ac02ab", + "workloads": [ + { + "dataset": "random/seed=862/workload-04", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 10000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "interval_secs": 10, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-04/q01-sum-fine/keys/600s" + ] + }, + { + "interval_secs": 10, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-04/q02-sum-coarse/keys/600s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-04/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-04/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-04/q05-quantile/values/100s" + ] + }, + { + "interval_secs": 2, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-04/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "interval_secs": 30, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-04/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-04/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-04/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-04/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-04/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 300000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 30000000.0, + "covered_min_N": 30000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-04", + "families": [ + { + "family": "topk", + "grid_K": 2500, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-04/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 4, + "keys_per_window": 2500 + }, + "lookback_secs": 1200.0, + "max_N": 30000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 30000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04-pilot/inference_config.yaml new file mode 100644 index 00000000..ec06c999 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04-pilot/inference_config.yaml @@ -0,0 +1,77 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service) (sum_over_time(data[10m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[10m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service, job) (sum_over_time(data[10m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[10m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 40 + query: topk by (region) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 30 + query: topk by (region, service) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04-pilot/plans.json.gz new file mode 100644 index 00000000..4c2bb7d6 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04-pilot/streaming_config.yaml new file mode 100644 index 00000000..636436db --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 10000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 10000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 2000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04.json index 2bf9ab31..26356bfb 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04.json +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-04.json @@ -1,4 +1,37 @@ { + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 10000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, "id": "workload-04", "labels": [ { @@ -23,6 +56,7 @@ ], "queries": [ { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -53,6 +87,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service", "grouping": [ "region", "service" @@ -81,15 +116,16 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", "grouping": [ "region" ], "id": "q03-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 4, "output_series_upper_bound": 4, - "promql": "quantile by (region) (0.75, data)", + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -98,8 +134,8 @@ "region" ], "id": "q03-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.75 } @@ -108,16 +144,17 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", "grouping": [ "region", "service" ], "id": "q04-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 20, "output_series_upper_bound": 20, - "promql": "quantile by (region, service) (0.75, data)", + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -127,8 +164,8 @@ "service" ], "id": "q04-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.75 } @@ -137,15 +174,16 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", "grouping": [ "region" ], "id": "q05-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 4, "output_series_upper_bound": 4, - "promql": "quantile by (region) (0.95, data)", + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -154,8 +192,8 @@ "region" ], "id": "q05-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.95 } @@ -164,6 +202,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", "grouping": [ "region", "service" @@ -193,6 +232,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -220,6 +260,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 10) * 10 - 600) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 10) * 10 - 600)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 10) * 10 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 10) * 10 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", "grouping": [ "region", "service" @@ -249,6 +290,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region", "grouping": [ "region" ], @@ -276,6 +318,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 10) * 10 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -299,7 +342,7 @@ "id": "q10-cardinality/rqe", "interval_ms": 10000, "lookback_ms": 600000, - "metric": "data", + "metric": "events", "value": "user_id" } ], diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05-pilot/eval_table.json new file mode 100644 index 00000000..1267f48b --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05-pilot/eval_table.json @@ -0,0 +1,445 @@ +{ + "inputs": { + "workload-05.json": "42bab8d17a1255bb1102aed686a9f05a600aac5fac79609e623545b535ac5a23" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "f885d7e9a99dd435683a7dcd66516a36b3ac02ab", + "workloads": [ + { + "dataset": "random/seed=862/workload-05", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 10000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "interval_secs": 10, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-05/q01-sum-fine/keys/300s" + ] + }, + { + "interval_secs": 10, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-05/q02-sum-coarse/keys/300s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-05/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + "random/seed=862/workload-05/q04-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-05/q05-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "topk by (region, service) (8, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-05/topk8_q06-topk-sum/keys/60s" + ] + }, + { + "interval_secs": 60, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-05/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 150000.0, + "covered_min_N": 150000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-05/q01-sum-fine/keys/300s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 150000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 150000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 1500000.0, + "covered_min_N": 1500000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-05/q02-sum-coarse/keys/300s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 1500000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 1500000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-05/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-05/topk8_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 300000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 30000000.0, + "covered_min_N": 30000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-05", + "families": [ + { + "family": "topk", + "grid_K": 2500, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-05/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 4, + "keys_per_window": 2500 + }, + "lookback_secs": 1200.0, + "max_N": 30000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 30000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05-pilot/inference_config.yaml new file mode 100644 index 00000000..e12768cc --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05-pilot/inference_config.yaml @@ -0,0 +1,79 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[5m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[5m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[5m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 20 + query: topk by (region) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 58 + query: topk by (region, service) (8, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.50, data[100s])) + planned_subquery: quantile_over_time(0.5, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05-pilot/plans.json.gz new file mode 100644 index 00000000..9d522ffc Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05-pilot/streaming_config.yaml new file mode 100644 index 00000000..3d3b4074 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 25000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 10000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 10000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 80 + width: 4096 + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 3000 + windowType: sliding +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05.json index f12afc78..c4776746 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05.json +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-05.json @@ -1,4 +1,37 @@ { + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 10000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, "id": "workload-05", "labels": [ { @@ -23,6 +56,7 @@ ], "queries": [ { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -53,6 +87,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service", "grouping": [ "region", "service" @@ -81,16 +116,17 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", "grouping": [ "region", "service" ], "id": "q03-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 20, "output_series_upper_bound": 20, - "promql": "quantile by (region, service) (0.99, data)", + "promql": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -100,8 +136,8 @@ "service" ], "id": "q03-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.99 } @@ -110,17 +146,18 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.50)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", "grouping": [ "region", "service", "job" ], "id": "q04-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 200, "output_series_upper_bound": 200, - "promql": "quantile by (region, service, job) (0.50, data)", + "promql": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -131,8 +168,8 @@ "job" ], "id": "q04-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.5 } @@ -141,15 +178,16 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", "grouping": [ "region" ], "id": "q05-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 4, "output_series_upper_bound": 4, - "promql": "quantile by (region) (0.90, data)", + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -158,8 +196,8 @@ "region" ], "id": "q05-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.9 } @@ -168,6 +206,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", "grouping": [ "region", "service" @@ -197,6 +236,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -224,6 +264,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 10) * 10 - 300) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 10) * 10 - 300)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 10) * 10 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 10) * 10 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", "grouping": [ "region", "service" @@ -253,6 +294,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region", "grouping": [ "region" ], @@ -280,6 +322,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -303,7 +346,7 @@ "id": "q10-cardinality/rqe", "interval_ms": 10000, "lookback_ms": 300000, - "metric": "data", + "metric": "events", "value": "user_id" } ], diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06-pilot/eval_table.json new file mode 100644 index 00000000..5aac84bb --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06-pilot/eval_table.json @@ -0,0 +1,398 @@ +{ + "inputs": { + "workload-06.json": "35bad66b98e305783fb612a45122e57aee7b0c6c96336ddc45e6e890e87d3ef1" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "f885d7e9a99dd435683a7dcd66516a36b3ac02ab", + "workloads": [ + { + "dataset": "random/seed=862/workload-06", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 10000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "interval_secs": 5, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-06/q01-sum-fine/keys/300s" + ] + }, + { + "interval_secs": 5, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-06/q02-sum-coarse/keys/300s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + "random/seed=862/workload-06/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + "random/seed=862/workload-06/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-06/q05-quantile/values/100s" + ] + }, + { + "interval_secs": 2, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-06/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "interval_secs": 60, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-06/topk8_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 150000.0, + "covered_min_N": 150000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-06/q01-sum-fine/keys/300s", + "instant": false, + "interval_secs": 5, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 150000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 150000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 1500000.0, + "covered_min_N": 1500000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-06/q02-sum-coarse/keys/300s", + "instant": false, + "interval_secs": 5, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 1500000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 1500000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-06/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-06/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 300000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-06", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-06/topk8_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06-pilot/inference_config.yaml new file mode 100644 index 00000000..d31f2ec0 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06-pilot/inference_config.yaml @@ -0,0 +1,79 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service) (sum_over_time(data[5m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[5m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 60 + query: sum by (region, service, job) (sum_over_time(data[5m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 30 + query: topk by (region, service) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 20 + query: topk by (region, service) (8, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service, job) (quantile_over_time(0.50, data[100s])) + planned_subquery: quantile_over_time(0.5, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.50, data[100s])) + planned_subquery: quantile_over_time(0.5, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06-pilot/plans.json.gz new file mode 100644 index 00000000..c91f5b9d Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06-pilot/streaming_config.yaml new file mode 100644 index 00000000..38466937 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 5000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 5000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 2000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06.json index ae2ad018..987a1b73 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06.json +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-06.json @@ -1,4 +1,37 @@ { + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 10000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, "id": "workload-06", "labels": [ { @@ -23,6 +56,7 @@ ], "queries": [ { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -53,6 +87,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service", "grouping": [ "region", "service" @@ -81,17 +116,18 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.50)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", "grouping": [ "region", "service", "job" ], "id": "q03-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 200, "output_series_upper_bound": 200, - "promql": "quantile by (region, service, job) (0.50, data)", + "promql": "avg by (region, service, job) (quantile_over_time(0.50, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -102,8 +138,8 @@ "job" ], "id": "q03-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.5 } @@ -112,16 +148,17 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.50)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", "grouping": [ "region", "service" ], "id": "q04-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 20, "output_series_upper_bound": 20, - "promql": "quantile by (region, service) (0.50, data)", + "promql": "avg by (region, service) (quantile_over_time(0.50, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -131,8 +168,8 @@ "service" ], "id": "q04-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.5 } @@ -141,15 +178,16 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", "grouping": [ "region" ], "id": "q05-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 4, "output_series_upper_bound": 4, - "promql": "quantile by (region) (0.90, data)", + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -158,8 +196,8 @@ "region" ], "id": "q05-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.9 } @@ -168,6 +206,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", "grouping": [ "region", "service" @@ -197,6 +236,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", "grouping": [ "region", "service" @@ -226,6 +266,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 5) * 5 - 300) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 5) * 5 - 300)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 5) * 5 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 5) * 5 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", "grouping": [ "region", "service" @@ -255,6 +296,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", "grouping": [ "region", "service" @@ -284,6 +326,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 5) * 5 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 5) * 5, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -307,7 +350,7 @@ "id": "q10-cardinality/rqe", "interval_ms": 5000, "lookback_ms": 300000, - "metric": "data", + "metric": "events", "value": "user_id" } ], diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07-pilot/eval_table.json new file mode 100644 index 00000000..fe799b3f --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07-pilot/eval_table.json @@ -0,0 +1,445 @@ +{ + "inputs": { + "workload-07.json": "b4f2431b6abad97dc834a1fd9bc27c9c008ce66efb6eaff79155063773ae23da" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "f885d7e9a99dd435683a7dcd66516a36b3ac02ab", + "workloads": [ + { + "dataset": "random/seed=862/workload-07", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 10000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "interval_secs": 20, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-07/q01-sum-fine/keys/600s" + ] + }, + { + "interval_secs": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-07/q02-sum-coarse/keys/600s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-07/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-07/q04-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.50, data[100s]))", + "rqes": [ + "random/seed=862/workload-07/q05-quantile/values/100s" + ] + }, + { + "interval_secs": 2, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-07/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "interval_secs": 60, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-07/topk8_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-07/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-07/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-07/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 1500000.0, + "covered_min_N": 1500000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "topk", + "grid_K": 2500, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-07/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 4, + "keys_per_window": 2500 + }, + "lookback_secs": 60.0, + "max_N": 1500000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 1500000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-07", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-07/topk8_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07-pilot/inference_config.yaml new file mode 100644 index 00000000..7832e815 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07-pilot/inference_config.yaml @@ -0,0 +1,80 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[10m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[10m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[10m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[10m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 30 + query: topk by (region) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 20 + query: topk by (region, service) (8, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service, job) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.50, data[100s])) + planned_subquery: quantile_over_time(0.5, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07-pilot/plans.json.gz new file mode 100644 index 00000000..ae91c0ab Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07-pilot/streaming_config.yaml new file mode 100644 index 00000000..17f3458b --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 25000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 20000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 2000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07.json index 5d7053cf..ef803fee 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07.json +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-07.json @@ -1,4 +1,37 @@ { + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 10000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, "id": "workload-07", "labels": [ { @@ -23,6 +56,7 @@ ], "queries": [ { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -53,6 +87,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service", "grouping": [ "region", "service" @@ -81,17 +116,18 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", "grouping": [ "region", "service", "job" ], "id": "q03-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 200, "output_series_upper_bound": 200, - "promql": "quantile by (region, service, job) (0.99, data)", + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -102,8 +138,8 @@ "job" ], "id": "q03-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.99 } @@ -112,17 +148,18 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", "grouping": [ "region", "service", "job" ], "id": "q04-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 200, "output_series_upper_bound": 200, - "promql": "quantile by (region, service, job) (0.95, data)", + "promql": "avg by (region, service, job) (quantile_over_time(0.95, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -133,8 +170,8 @@ "job" ], "id": "q04-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.95 } @@ -143,15 +180,16 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.50)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", "grouping": [ "region" ], "id": "q05-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 4, "output_series_upper_bound": 4, - "promql": "quantile by (region) (0.50, data)", + "promql": "avg by (region) (quantile_over_time(0.50, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -160,8 +198,8 @@ "region" ], "id": "q05-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.5 } @@ -170,6 +208,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -197,6 +236,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", "grouping": [ "region", "service" @@ -226,6 +266,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 20) * 20 - 600) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 20) * 20 - 600)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 20) * 20 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 20) * 20 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -253,6 +294,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", "grouping": [ "region", "service" @@ -282,6 +324,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region", "grouping": [ "region" ], @@ -301,7 +344,7 @@ "id": "q10-cardinality/rqe", "interval_ms": 20000, "lookback_ms": 600000, - "metric": "data", + "metric": "events", "value": "user_id" } ], diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08-pilot/eval_table.json new file mode 100644 index 00000000..5f708f70 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08-pilot/eval_table.json @@ -0,0 +1,392 @@ +{ + "inputs": { + "workload-08.json": "622e6659804f4320c46f1c7a8bd5c333834672d0b80dc1b9324bb117fe32afc8" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "f885d7e9a99dd435683a7dcd66516a36b3ac02ab", + "workloads": [ + { + "dataset": "random/seed=862/workload-08", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 10000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "interval_secs": 10, + "promql": "sum by (region, service, job) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-08/q01-sum-fine/keys/300s" + ] + }, + { + "interval_secs": 10, + "promql": "sum by (region, service) (sum_over_time(data[5m]))", + "rqes": [ + "random/seed=862/workload-08/q02-sum-coarse/keys/300s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-08/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-08/q04-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-08/q04-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "topk by (region) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-08/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "interval_secs": 30, + "promql": "topk by (region) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-08/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 150000.0, + "covered_min_N": 150000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-08/q01-sum-fine/keys/300s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 150000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 150000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 1500000.0, + "covered_min_N": 1500000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-08/q02-sum-coarse/keys/300s", + "instant": false, + "interval_secs": 10, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 300.0, + "max_N": 1500000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 1500000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "300s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-08/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 1500000.0, + "covered_min_N": 1500000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "topk", + "grid_K": 2500, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-08/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 4, + "keys_per_window": 2500 + }, + "lookback_secs": 60.0, + "max_N": 1500000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 1500000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 30000000.0, + "covered_min_N": 30000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-08", + "families": [ + { + "family": "topk", + "grid_K": 2500, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-08/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 4, + "keys_per_window": 2500 + }, + "lookback_secs": 1200.0, + "max_N": 30000000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 30000000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08-pilot/inference_config.yaml new file mode 100644 index 00000000..85723e3e --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08-pilot/inference_config.yaml @@ -0,0 +1,78 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[5m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[5m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[5m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[5m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 59 + query: topk by (region) (4, sum_over_time(data[1m])) + planned_subquery: topk by (region) (4, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 40 + query: topk by (region) (4, sum_over_time(data[20m])) + planned_subquery: topk by (region) (4, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08-pilot/plans.json.gz new file mode 100644 index 00000000..bc0b9a01 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08-pilot/streaming_config.yaml new file mode 100644 index 00000000..bb94015b --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 10000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 10000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 120 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: sliding +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08.json index dd6f1df6..aca22cb7 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08.json +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-08.json @@ -1,4 +1,37 @@ { + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 10000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, "id": "workload-08", "labels": [ { @@ -23,6 +56,7 @@ ], "queries": [ { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -53,6 +87,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service", "grouping": [ "region", "service" @@ -81,16 +116,17 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", "grouping": [ "region", "service" ], "id": "q03-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 20, "output_series_upper_bound": 20, - "promql": "quantile by (region, service) (0.75, data)", + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -100,8 +136,8 @@ "service" ], "id": "q03-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.75 } @@ -110,16 +146,17 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", "grouping": [ "region", "service" ], "id": "q04-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 20, "output_series_upper_bound": 20, - "promql": "quantile by (region, service) (0.95, data)", + "promql": "avg by (region, service) (quantile_over_time(0.95, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -129,8 +166,8 @@ "service" ], "id": "q04-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.95 } @@ -139,15 +176,16 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", "grouping": [ "region" ], "id": "q05-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 4, "output_series_upper_bound": 4, - "promql": "quantile by (region) (0.95, data)", + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -156,8 +194,8 @@ "region" ], "id": "q05-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.95 } @@ -166,6 +204,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -193,6 +232,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -220,6 +260,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 10) * 10 - 300) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 10) * 10 - 300)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 10) * 10 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 10) * 10 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region", "grouping": [ "region" ], @@ -247,6 +288,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -274,6 +316,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 10) * 10 - 300, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 10) * 10, 3) GROUP BY region", "grouping": [ "region" ], @@ -293,7 +336,7 @@ "id": "q10-cardinality/rqe", "interval_ms": 10000, "lookback_ms": 300000, - "metric": "data", + "metric": "events", "value": "user_id" } ], diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09-pilot/eval_table.json new file mode 100644 index 00000000..68f526ec --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09-pilot/eval_table.json @@ -0,0 +1,398 @@ +{ + "inputs": { + "workload-09.json": "b5050e8d5ef16a3d4c7a14e6396e40b0e8bcb40c0eb4d56b1ca4959d1b8a40f7" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "f885d7e9a99dd435683a7dcd66516a36b3ac02ab", + "workloads": [ + { + "dataset": "random/seed=862/workload-09", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 10000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "interval_secs": 2, + "promql": "sum by (region, service, job) (sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-09/q01-sum-fine/keys/60s" + ] + }, + { + "interval_secs": 2, + "promql": "sum by (region, service) (sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-09/q02-sum-coarse/keys/60s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-09/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-09/q04-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-09/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "topk by (region, service) (8, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-09/topk8_q06-topk-sum/keys/60s" + ] + }, + { + "interval_secs": 30, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-09/topk8_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 30000.0, + "covered_min_N": 30000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-09/q01-sum-fine/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 60.0, + "max_N": 30000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 30000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-09/q02-sum-coarse/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 60.0, + "max_N": 300000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-09/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-09/topk8_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 300000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-09", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-09/topk8_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09-pilot/inference_config.yaml new file mode 100644 index 00000000..45c15960 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09-pilot/inference_config.yaml @@ -0,0 +1,81 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[1m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[1m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[1m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 58 + query: topk by (region, service) (8, sum_over_time(data[1m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 40 + query: topk by (region, service) (8, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service, job) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service, job) (quantile_over_time(0.95, data[100s])) + planned_subquery: quantile_over_time(0.95, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + - job + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 81 + query: avg by (region, service) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09-pilot/plans.json.gz new file mode 100644 index 00000000..1a3e2542 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09-pilot/streaming_config.yaml new file mode 100644 index 00000000..79c184c5 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 2000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 1024 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 3000 + windowType: sliding +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 320 + width: 4096 + slideIntervalMs: 30000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 30000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09.json index f1c095f4..f984d0cd 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09.json +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-09.json @@ -1,4 +1,37 @@ { + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 10000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, "id": "workload-09", "labels": [ { @@ -23,6 +56,7 @@ ], "queries": [ { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -53,6 +87,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service", "grouping": [ "region", "service" @@ -81,17 +116,18 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", "grouping": [ "region", "service", "job" ], "id": "q03-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 200, "output_series_upper_bound": 200, - "promql": "quantile by (region, service, job) (0.75, data)", + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -102,8 +138,8 @@ "job" ], "id": "q03-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.75 } @@ -112,17 +148,18 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.95)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service, job", "grouping": [ "region", "service", "job" ], "id": "q04-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 200, "output_series_upper_bound": 200, - "promql": "quantile by (region, service, job) (0.95, data)", + "promql": "avg by (region, service, job) (quantile_over_time(0.95, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -133,8 +170,8 @@ "job" ], "id": "q04-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.95 } @@ -143,16 +180,17 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", "grouping": [ "region", "service" ], "id": "q05-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 20, "output_series_upper_bound": 20, - "promql": "quantile by (region, service) (0.75, data)", + "promql": "avg by (region, service) (quantile_over_time(0.75, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -162,8 +200,8 @@ "service" ], "id": "q05-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.75 } @@ -172,6 +210,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", "grouping": [ "region", "service" @@ -201,6 +240,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", "grouping": [ "region", "service" @@ -230,6 +270,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 2) * 2 - 60) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 2) * 2 - 60)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 2) * 2 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 2) * 2 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region", "grouping": [ "region" ], @@ -257,6 +298,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 30) * 30 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 30) * 30 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 30) * 30 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 30) * 30 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 30) * 30 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 30) * 30, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", "grouping": [ "region", "service" @@ -286,6 +328,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service", "grouping": [ "region", "service" @@ -307,7 +350,7 @@ "id": "q10-cardinality/rqe", "interval_ms": 2000, "lookback_ms": 60000, - "metric": "data", + "metric": "events", "value": "user_id" } ], diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10-pilot/eval_table.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10-pilot/eval_table.json new file mode 100644 index 00000000..439172d8 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10-pilot/eval_table.json @@ -0,0 +1,445 @@ +{ + "inputs": { + "workload-10.json": "6bced7e10708a518ebbbc6ecb144177337748c080bf2e44f16e471485532a3b4" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "f885d7e9a99dd435683a7dcd66516a36b3ac02ab", + "workloads": [ + { + "dataset": "random/seed=862/workload-10", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 10000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "interval_secs": 20, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-10/q01-sum-fine/keys/600s" + ] + }, + { + "interval_secs": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-10/q02-sum-coarse/keys/600s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-10/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-10/q04-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", + "rqes": [ + "random/seed=862/workload-10/q05-quantile/values/100s" + ] + }, + { + "interval_secs": 2, + "promql": "topk by (region) (8, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-10/topk8_q06-topk-sum/keys/60s" + ] + }, + { + "interval_secs": 60, + "promql": "topk by (region, service) (8, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-10/topk8_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-10/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-10/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-10/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "topk", + "covered_groups": 4, + "covered_max_N": 1500000.0, + "covered_min_N": 1500000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "topk", + "grid_K": 2500, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region" + ], + "id": "random/seed=862/workload-10/topk8_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 2, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 4, + "keys_per_window": 2500 + }, + "lookback_secs": 60.0, + "max_N": 1500000.0, + "max_covered_share": 0.25, + "metric": "data", + "min_N": 1500000.0, + "min_covered_share": 0.25, + "queries_per_instance": 1, + "query_id": "topk8_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-10", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-10/topk8_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 60, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk8_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10-pilot/inference_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10-pilot/inference_config.yaml new file mode 100644 index 00000000..f9dc1204 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10-pilot/inference_config.yaml @@ -0,0 +1,77 @@ +cleanup_policy: + name: circular_buffer +metrics: + data: + - job + - label_0 + - region + - service +queries: +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service) (sum_over_time(data[10m])) + planned_subquery: sum by (job, region, service) (sum_over_time(data[10m])) + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 30 + query: sum by (region, service, job) (sum_over_time(data[10m])) + planned_subquery: sum by (region, service, job) (sum_over_time(data[10m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 3 + num_aggregates_to_retain: 30 + query: topk by (region) (8, sum_over_time(data[1m])) + planned_subquery: topk by (region) (8, sum_over_time(data[1m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 4 + num_aggregates_to_retain: 20 + query: topk by (region, service) (8, sum_over_time(data[20m])) + planned_subquery: topk by (region, service) (8, sum_over_time(data[20m])) + query_time_aggregations: [] +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region, service) (quantile_over_time(0.99, data[100s])) + planned_subquery: quantile_over_time(0.99, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + - service + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.75, data[100s])) + planned_subquery: quantile_over_time(0.75, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null +- aggregations: + - aggregation_id: 1 + num_aggregates_to_retain: 76 + query: avg by (region) (quantile_over_time(0.90, data[100s])) + planned_subquery: quantile_over_time(0.9, data[1m40s]) + query_time_aggregations: + - operator: avg + grouping: + mode: by + labels: + - region + parameter: null diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10-pilot/plans.json.gz b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10-pilot/plans.json.gz new file mode 100644 index 00000000..79a48ed5 Binary files /dev/null and b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10-pilot/plans.json.gz differ diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10-pilot/streaming_config.yaml b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10-pilot/streaming_config.yaml new file mode 100644 index 00000000..894cdbb6 --- /dev/null +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10-pilot/streaming_config.yaml @@ -0,0 +1,89 @@ +aggregations: +- aggregationId: 1 + aggregationSubType: '' + aggregationType: DDSketch + labels: + aggregated: [] + grouping: + - job + - label_0 + - region + - service + rollup: [] + metric: data + parameters: + alpha: 0.1 + slideIntervalMs: 1000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 25000 + windowType: sliding +- aggregationId: 2 + aggregationSubType: sum + aggregationType: MultipleSum + labels: + aggregated: + - job + - region + - service + grouping: [] + rollup: [] + metric: data + parameters: {} + slideIntervalMs: 20000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 20000 + windowType: tumbling +- aggregationId: 3 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + - service + grouping: + - region + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 240 + width: 1024 + slideIntervalMs: 2000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 2000 + windowType: tumbling +- aggregationId: 4 + aggregationSubType: sum + aggregationType: CountMinSketchWithHeap + labels: + aggregated: + - job + - label_0 + grouping: + - region + - service + rollup: [] + metric: data + parameters: + depth: 3 + heapsize: 160 + width: 4096 + slideIntervalMs: 60000 + spatialFilter: '' + table_name: null + value_column: null + windowSizeMs: 60000 + windowType: tumbling +metrics: + data: + - job + - label_0 + - region + - service diff --git a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10.json b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10.json index d93c5ac3..59f05c09 100644 --- a/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10.json +++ b/asap-tools/experiments/datasets/fake_metric/random_workloads/seed-862/workload-10.json @@ -1,4 +1,37 @@ { + "data_facts": { + "backfill_history_ms": 1200000, + "events": { + "assignment": "deterministic_rotating_pool_with_cross_leaf_overlap", + "distinct_key": "user_id", + "table": "events", + "time_column": "ts", + "user_id_universe_size": 10000, + "user_ids_per_leaf": 1000 + }, + "feeder_seed": 862, + "metrics": { + "data": { + "kind": "gauge", + "pareto_shape": 2.0, + "value_shape": "rank_scaled_pareto" + }, + "data_total": { + "increment_shape": "rank_scaled_positive_deterministic", + "kind": "counter", + "resets": "none" + } + }, + "scrape_interval_ms": 100, + "series": { + "leaf_count": 10000, + "ranking": { + "exponent": 1.1, + "kind": "deterministic_zipf_like", + "rank_scope": "leaf_series" + } + } + }, "id": "workload-10", "labels": [ { @@ -23,6 +56,7 @@ ], "queries": [ { + "clickhouse_sql": "SELECT region, service, job, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job", "grouping": [ "region", "service", @@ -53,6 +87,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service", "grouping": [ "region", "service" @@ -81,16 +116,17 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.99)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region, service", "grouping": [ "region", "service" ], "id": "q03-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 20, "output_series_upper_bound": 20, - "promql": "quantile by (region, service) (0.99, data)", + "promql": "avg by (region, service) (quantile_over_time(0.99, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -100,8 +136,8 @@ "service" ], "id": "q03-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.99 } @@ -110,15 +146,16 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.75)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", "grouping": [ "region" ], "id": "q04-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 4, "output_series_upper_bound": 4, - "promql": "quantile by (region) (0.75, data)", + "promql": "avg by (region) (quantile_over_time(0.75, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -127,8 +164,8 @@ "region" ], "id": "q04-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.75 } @@ -137,15 +174,16 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, avg(value) AS value FROM (SELECT region, service, job, label_0, quantileExactInclusive(0.90)(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 1) * 1 - 100, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 1) * 1, 3) GROUP BY region, service, job, label_0) GROUP BY region", "grouping": [ "region" ], "id": "q05-quantile", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "output_groups": 4, "output_series_upper_bound": 4, - "promql": "quantile by (region) (0.90, data)", + "promql": "avg by (region) (quantile_over_time(0.90, data[100s]))", "rqes": [ { "accuracy_sla": 0.95, @@ -154,8 +192,8 @@ "region" ], "id": "q05-quantile/rqe", - "interval_ms": 100, - "lookback_ms": 100, + "interval_ms": 1000, + "lookback_ms": 100000, "metric": "data", "phi": 0.9 } @@ -164,6 +202,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 2) * 2 - 60, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 2) * 2, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region", "grouping": [ "region" ], @@ -191,6 +230,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, sum(value) AS value FROM data WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", "grouping": [ "region", "service" @@ -220,6 +260,7 @@ "support": "common-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 20) * 20 - 600) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 20) * 20 - 600)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 20) * 20 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 20) * 20 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 8 BY region, service", "grouping": [ "region", "service" @@ -249,6 +290,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, service, job, label_0, delta * (sampled + to_start + to_end) / sampled / 1200 AS value FROM (SELECT region, service, job, label_0, last_v - first_v AS delta, first_v, last_t - first_t AS sampled, sampled / (n - 1) AS avg_step, if(first_t - (intDiv({{ end_time }}, 60) * 60 - 1200) >= 1.1 * avg_step, avg_step / 2, first_t - (intDiv({{ end_time }}, 60) * 60 - 1200)) AS to_start_clamped, if(delta > 0 AND first_v >= 0, least(to_start_clamped, sampled * first_v / delta), to_start_clamped) AS to_start, if(intDiv({{ end_time }}, 60) * 60 - last_t >= 1.1 * avg_step, avg_step / 2, intDiv({{ end_time }}, 60) * 60 - last_t) AS to_end FROM (SELECT region, service, job, label_0, argMin(value, ts) AS first_v, argMax(value, ts) AS last_v, toFloat64(min(ts)) AS first_t, toFloat64(max(ts)) AS last_t, count() AS n FROM data_total WHERE ts >= toDateTime64(intDiv({{ end_time }}, 60) * 60 - 1200, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 60) * 60, 3) GROUP BY region, service, job, label_0 HAVING n >= 2)) ORDER BY value DESC, region, service, job, label_0 LIMIT 4 BY region, service", "grouping": [ "region", "service" @@ -278,6 +320,7 @@ "support": "counter-gated-promql" }, { + "clickhouse_sql": "SELECT region, uniqExact(user_id) AS value FROM events WHERE ts > toDateTime64(intDiv({{ end_time }}, 20) * 20 - 600, 3) AND ts <= toDateTime64(intDiv({{ end_time }}, 20) * 20, 3) GROUP BY region", "grouping": [ "region" ], @@ -297,7 +340,7 @@ "id": "q10-cardinality/rqe", "interval_ms": 20000, "lookback_ms": 600000, - "metric": "data", + "metric": "events", "value": "user_id" } ], diff --git a/asap-tools/experiments/datasets/fake_metric/sketch_bench_workloads.py b/asap-tools/experiments/datasets/fake_metric/sketch_bench_workloads.py index adeb011b..01fd972f 100644 --- a/asap-tools/experiments/datasets/fake_metric/sketch_bench_workloads.py +++ b/asap-tools/experiments/datasets/fake_metric/sketch_bench_workloads.py @@ -126,11 +126,13 @@ class Query: With `series_fn` None, `agg` runs over the raw samples of each `grouping` group in the window (sum_over_time, sum by (job) (sum_over_time), ...). - Otherwise `series_fn` (last, rate, increase) gives one value per series, - and `agg` combines those per group; `agg` None keeps them per series. + Otherwise `series_fn` (last, rate, increase, or quantile at `quantile`) + gives one value per series, and `agg` combines those per group (avg by + (job) (quantile_over_time)); `agg` None keeps them per series. `ratio_quantile` divides the `quantile` by a second quantile of the same - samples. Structured (non-PromQL) query entries map onto these fields and - set `left_open`: their window is (end - W, end], not PromQL's closed one. + samples. `topk` keeps the k largest values, per `topk_by` group when set. + Structured (non-PromQL) query entries map onto these fields and set + `left_open`: their window is (end - W, end], not PromQL's closed one. """ table: str @@ -142,6 +144,7 @@ class Query: quantile: Optional[str] = None # level as written, e.g. "0.99" ratio_quantile: Optional[str] = None topk: Optional[int] = None + topk_by: Tuple[str, ...] = () value: str = VALUE_COLUMN left_open: bool = False @@ -297,10 +300,9 @@ def render( # The grand total: one row, none for an empty window. sql = f"SELECT {value} AS value {window} HAVING count() > 0" else: - if q.series_fn == "last": - sql = ( - f"SELECT {cols}, {agg('last', None)} AS value {window} GROUP BY {cols}" - ) + if q.series_fn in ("last", "quantile"): + value = agg(q.series_fn, q.quantile) + sql = f"SELECT {cols}, {value} AS value {window} GROUP BY {cols}" else: samples = ( f"SELECT {cols}, {agg('counter', None)} {window} " @@ -313,6 +315,8 @@ def render( sql = f"SELECT {group}, {combine} AS value FROM ({sql}) GROUP BY {group}" if q.topk is not None: sql += f" ORDER BY value DESC, {group} LIMIT {q.topk}" + if q.topk_by: + sql += f" BY {', '.join(q.topk_by)}" return sql @@ -321,6 +325,8 @@ def raw_agg(stat: str, quantile: Optional[str], value: str = VALUE_COLUMN) -> st return f"sum({value})" if stat == "count": return "count()" + if stat == "avg": + return f"avg({value})" if stat == "cardinality": return f"uniqExact({value})" if stat == "quantile": @@ -413,7 +419,9 @@ class View: def view_stat(q: Query) -> str: if q.series_fn is None: return q.agg - return "last" if q.series_fn == "last" else "counter" + if q.series_fn in ("last", "quantile"): + return q.series_fn + return "counter" def mv_servable(q: Query) -> bool: @@ -425,7 +433,8 @@ def plan_views(queries: List[Query], metrics: dict) -> Dict[tuple, View]: """One view per (table, value, statistic, interval), keyed by view_key(). Pooled statistics keep the union of their queries' groupings plus every label's child_of parents, so coarser groupings roll up; - per-series statistics keep every label.""" + per-series statistics, and any view a per-series query reads, keep every + label.""" grouped: Dict[tuple, List[Query]] = {} for q in queries: if not mv_servable(q): @@ -442,7 +451,8 @@ def plan_views(queries: List[Query], metrics: dict) -> Dict[tuple, View]: unknown = {p for p in parent.values() if p} - set(parent) if unknown: raise ValueError(f"{table!r}: child_of names unknown labels {unknown}") - keep = set(parent) if stat in PER_SERIES_STATS else set() + per_series = stat in PER_SERIES_STATS or any(q.series_fn for q in qs) + keep = set(parent) if per_series else set() for name in (n for q in qs for n in q.grouping): while name and name not in keep: keep.add(name) diff --git a/asap-tools/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2 b/asap-tools/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2 index adf14db5..c47bec7f 100644 --- a/asap-tools/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2 +++ b/asap-tools/experiments/experiment_utils/services/docker-compose.victoriametrics.yml.j2 @@ -19,6 +19,9 @@ services: # Raise the per-query caps so 24 h windows over many series aren't rejected. - "-search.maxSamplesPerQuery=10000000000" - "-search.maxPointsPerTimeseries=1000000" + # Serve samples as soon as they arrive, as Prometheus does; the default + # 30 s hides the end of any window queried sooner after it closes. + - "-search.latencyOffset=0s" {% if cpu_limit or memory_limit -%} deploy: resources: diff --git a/asap-tools/experiments/post_experiment/multi_experiment/compare_arms.py b/asap-tools/experiments/post_experiment/multi_experiment/compare_arms.py new file mode 100644 index 00000000..dc7bf986 --- /dev/null +++ b/asap-tools/experiments/post_experiment/multi_experiment/compare_arms.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +"""Compare one workload's arms, each a mode of its own experiment run. + +For every query: latency (p50/p95/p99) and the share answered within its +repeat interval; each arm's agreement with a reference arm (mean relative +error, or precision@k for top-k); and per arm the total CPU-seconds and peak +memory from remote_monitor's samples. Arms must have run the same queries in +the same order on the same data (same feeder seed), so (query group, query, +repetition) names the same window in every arm. + +Usage: + python3 compare_arms.py --reference prometheus \\ + --arm asapquery=experiment_outputs/w01_asap/sketchdb \\ + --arm prometheus=experiment_outputs/w01_prom/baseline \\ + --arm clickhouse_mv=experiment_outputs/w01_ch_mv/baseline_mv +""" + +import argparse +import gzip +import json +import statistics +from collections import defaultdict +from pathlib import Path + +import yaml + + +def read_jsonl(path): + with gzip.open(path, "rt") as f: + return [json.loads(line) for line in f if line.strip()] + + +def sql_labels(sql): + """The label columns a ClickHouse statement selects before its value, + after any leading `-- description` line.""" + lines = sql.splitlines() + body = "\n".join(lines[1:]) if lines and lines[0].startswith("--") else sql + cols = [] + for token in body[len("SELECT ") :].split(", "): + if "(" in token: + break + cols.append(token.strip()) + return cols + + +def query_name(query): + """PromQL as is; ClickHouse SQL by its `-- ` description line.""" + first = query.splitlines()[0] + return first[2:].strip() if first.startswith("--") else query + + +class Arm: + """One experiment mode's latencies, results, intervals and resources.""" + + def __init__(self, mode_dir): + mode_dir = Path(mode_dir) + out = mode_dir / "prometheus_client_output" + self.latencies = defaultdict(list) # (group, query) -> [seconds] + for r in read_jsonl(out / "query_latencies.jsonl.gz"): + self.latencies[(r["query_group_idx"], r["query_idx"])].append(r["latency"]) + self.results = defaultdict(dict) # (group, query, rep) -> {labels: value} + self.names = {} + for r in read_jsonl(out / "query_results.jsonl.gz"): + key = (r["query_group_idx"], r["query_idx"], r["repetition_idx"]) + self.names[key[:2]] = query_name(r["query"]) + window = self.results[key] + if "raw_text_result" in r: + cols = sql_labels(r["query"]) + for line in (r["raw_text_result"] or "").strip().splitlines(): + *labels, value = line.split("\t") + window[tuple(sorted(zip(cols, labels)))] = float(value) + elif r.get("result_labels") is not None: + labels = json.loads(r["result_labels"]) + labels.pop("__name__", None) + window[tuple(sorted(labels.items()))] = float(r["result_value"]) + client = mode_dir.parent / "controller_client_configs" / f"{mode_dir.name}.yaml" + groups = yaml.safe_load(client.read_text())["query_groups"] + self.interval_secs = [g["repetition_delay_ms"] / 1000 for g in groups] + self.monitor = mode_dir / "remote_monitor_output" / "monitor_output.json" + hydra = yaml.safe_load((mode_dir.parent / "hydra_config.yaml").read_text()) + self.monitor_interval_secs = float(hydra["flow"]["monitor_interval_seconds"]) + + def resources(self): + """(total CPU-seconds, peak memory bytes), summed over monitored + processes at each sample.""" + processes = json.loads(self.monitor.read_text()).values() + n = max(len(p["cpu_percent"]) for p in processes) + cpu, mem = [0.0] * n, [0.0] * n + for p in processes: + for t, c in enumerate(p["cpu_percent"]): + cpu[t] += c + for t, m in enumerate(p["memory_info"]): + mem[t] += m + return sum(cpu) / 100 * self.monitor_interval_secs, max(mem) + + +def percentile(xs, p): + xs = sorted(xs) + return xs[min(len(xs) - 1, int(p / 100 * len(xs)))] + + +def agreement(got, want, topk): + """`got`'s window against reference `want`: the share of `want`'s series + `got` returns (top-k), else the mean relative error over shared series + and the number of `want`'s series `got` lacks.""" + if topk: + return len(set(got) & set(want)) / len(want), 0 + errors = [abs(got[k] - v) / abs(v) for k, v in want.items() if k in got and v] + return (statistics.mean(errors) if errors else None), len(set(want) - set(got)) + + +def report(arms, reference): + names = arms[reference].names + print("== Latency (s): p50 / p95 / p99, share within the repeat interval") + for gq, name in sorted(names.items()): + interval = arms[reference].interval_secs[gq[0]] + print(f"\n[{gq[0]}.{gq[1]}] every {interval:g}s: {name}") + for arm_name, arm in arms.items(): + xs = arm.latencies.get(gq) + if not xs: + continue + within = sum(x <= interval for x in xs) / len(xs) + print( + f" {arm_name:16s} n={len(xs):5d} {percentile(xs, 50):8.3f} " + f"{percentile(xs, 95):8.3f} {percentile(xs, 99):8.3f} {within:6.1%}" + ) + + print(f"\n== Agreement with {reference}") + ref = arms[reference].results + for gq, name in sorted(names.items()): + topk = name.startswith("topk") + print(f"\n[{gq[0]}.{gq[1]}] {name}") + for arm_name, arm in arms.items(): + if arm_name == reference: + continue + values, missing, empty = [], 0, 0 + for key, want in ref.items(): + if key[:2] != gq or key not in arm.results: + continue + if not want: + empty += 1 + continue + value, lacking = agreement(arm.results[key], want, topk) + missing += lacking + if value is not None: + values.append(value) + if not values: + print(f" {arm_name:16s} no comparable windows") + continue + worst = min(values) if topk else max(values) + what = "precision@k" if topk else "mean rel err" + print( + f" {arm_name:16s} {what} {statistics.mean(values):.3g} " + f"(worst {worst:.3g}) over {len(values)} windows; " + f"missing series {missing}, empty reference windows {empty}" + ) + + print("\n== Resources (whole monitored run)") + for arm_name, arm in arms.items(): + cpu_secs, peak = arm.resources() + print( + f" {arm_name:16s} CPU {cpu_secs:9.0f} s peak memory {peak / 2**30:5.2f} GiB" + ) + + +def main(): + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--arm", + action="append", + required=True, + metavar="NAME=MODE_DIR", + help="an arm: its name and its experiment's mode directory", + ) + parser.add_argument( + "--reference", required=True, help="the arm others are scored against" + ) + args = parser.parse_args() + arms = {} + for spec in args.arm: + name, _, mode_dir = spec.partition("=") + arms[name] = Arm(mode_dir) + if args.reference not in arms: + parser.error(f"--reference {args.reference!r} is not an --arm") + report(arms, args.reference) + + +if __name__ == "__main__": + main() diff --git a/asap-tools/experiments/tests/test_compare_arms.py b/asap-tools/experiments/tests/test_compare_arms.py new file mode 100644 index 00000000..99c5ac33 --- /dev/null +++ b/asap-tools/experiments/tests/test_compare_arms.py @@ -0,0 +1,40 @@ +"""Tests for post_experiment/multi_experiment/compare_arms.py.""" + +import os +import sys +import unittest + +sys.path.insert( + 0, + os.path.join( + os.path.dirname(__file__), "..", "post_experiment", "multi_experiment" + ), +) + +import compare_arms # noqa: E402 + + +class CompareArmsTest(unittest.TestCase): + def test_sql_labels_skip_the_description_and_stop_at_the_value(self): + sql = ( + "-- topk by (region) (4, sum_over_time(data[1m]))\n" + "SELECT region, label_0, sum(value) AS value FROM data GROUP BY region, label_0" + ) + self.assertEqual(compare_arms.sql_labels(sql), ["region", "label_0"]) + self.assertEqual( + compare_arms.query_name(sql), + "topk by (region) (4, sum_over_time(data[1m]))", + ) + + def test_agreement_scores_values_and_topk_sets(self): + a, b = (("region", "0"),), (("region", "1"),) + self.assertEqual( + compare_arms.agreement({a: 11.0}, {a: 10.0, b: 5.0}, False), (0.1, 1) + ) + self.assertEqual( + compare_arms.agreement({a: 1.0}, {a: 9.0, b: 8.0}, True), (0.5, 0) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/asap-tools/experiments/tests/test_random_workload_artifacts.py b/asap-tools/experiments/tests/test_random_workload_artifacts.py index a8683697..35ef2be6 100644 --- a/asap-tools/experiments/tests/test_random_workload_artifacts.py +++ b/asap-tools/experiments/tests/test_random_workload_artifacts.py @@ -1,9 +1,24 @@ """Tests for manifest-to-planner workload rendering.""" +import math import os +import random +import shutil +import subprocess import sys +import json +import tempfile import unittest +from pathlib import Path +import yaml +from hydra import compose, initialize_config_dir +from omegaconf import OmegaConf + +from experiment_run_clickhouse import _check_clickhouse_paced_feeder +from experiment_utils import config as experiment_config +from experiment_utils.core import read_exporter_config +from experiment_utils.services.paced_feeder import check_paced_feeder_config sys.path.insert( 0, os.path.join(os.path.dirname(__file__), "..", "datasets", "fake_metric") @@ -11,11 +26,25 @@ import random_workload_artifacts as artifacts # noqa: E402 import random_workloads as workloads # noqa: E402 +import sketch_bench_workloads as sbw # noqa: E402 + +PILOT_DIR = ( + Path(__file__).parent.parent + / "datasets/fake_metric/random_workloads/seed-862/workload-01-pilot" +) +CLICKHOUSE = shutil.which("clickhouse") + + +def pilot(): + workload = workloads.generate_workloads(seed=862)[0] + files = artifacts.pilot_configs(workload, PILOT_DIR) + return workload, {path.name: content for path, content in files.items()} class RenderArtifactsTest(unittest.TestCase): def test_promql_input_groups_non_sql_queries_by_repeat_interval(self): - rendered = artifacts.render(workloads.generate_workloads(seed=862)[0]) + workload = workloads.generate_workloads(seed=862)[0] + rendered = artifacts.render(workload) planner = rendered["promql_planner"] expected = [q for q in rendered["queries"] if q["promql"]] @@ -27,6 +56,7 @@ def test_promql_input_groups_non_sql_queries_by_repeat_interval(self): {tuple(metric["labels"]) for metric in planner["metrics"]}, {("region", "service", "job", "label_0")}, ) + self.assertEqual(rendered["data_facts"], workload["data_facts"]) for group in planner["query_groups"]: self.assertTrue( all( @@ -48,6 +78,254 @@ def test_sql_input_contains_only_sql_queries_and_their_table_schema(self): ] self.assertTrue(all("COUNT(DISTINCT user_id)" in query for query in sql)) + def test_paced_feeder_table_preserves_all_manifest_data_facts(self): + table = artifacts.paced_feeder_table(workloads.generate_workloads(seed=862)[0]) + metrics = table["workloads"][0]["e2e"]["metrics"] + + self.assertEqual(metrics["data"]["samples_per_sec_per_series"], 10) + self.assertEqual(metrics["data"]["values"]["value"]["keys"], 10_000) + self.assertEqual(metrics["data_total"]["type"], "counter") + self.assertEqual(metrics["events"]["records_per_sec"], 100_000) + self.assertEqual( + metrics["events"]["values"]["user_id"]["rotating_pool"], + {"universe_keys": 10_000, "keys_per_series": 1_000}, + ) + + def test_archived_pilot_plans_read_the_manifests_pilot_queries(self): + # An archived plan is only valid for the queries it was solved for. + for workload in workloads.generate_workloads(seed=862): + with self.subTest(workload=workload["id"]): + pilot_dir = PILOT_DIR.parent / f"{workload['id']}-pilot" + with open(pilot_dir / "eval_table.json") as f: + e2e = json.load(f)["workloads"][0]["e2e"] + self.assertEqual( + [(q["promql"], q["interval_secs"] * 1000) for q in e2e["queries"]], + [ + (q["promql"], q["interval_ms"]) + for q in workload["queries"] + if q["support"] == "common-promql" + ], + ) + # The planner's configs answer every pilot query. + inference = yaml.safe_load( + (pilot_dir / "inference_config.yaml").read_text() + ) + self.assertEqual( + sorted(q["query"] for q in inference["queries"]), + sorted(q["promql"] for q in e2e["queries"]), + ) + + def test_every_workload_renders_its_pilot_configs(self): + for workload in workloads.generate_workloads(seed=862): + with self.subTest(workload=workload["id"]): + pilot_dir = PILOT_DIR.parent / f"{workload['id']}-pilot" + files = artifacts.pilot_configs(workload, pilot_dir) + self.assertEqual( + sum(path.suffix == ".yaml" for path in files), 4, sorted(files) + ) + + +class PilotConfigsTest(unittest.TestCase): + def test_every_arm_feeds_the_same_data_and_schedule(self): + workload, files = pilot() + e2e = yaml.safe_load(files["e2e.yaml"])["experiment_params"] + vm = yaml.safe_load(files["victoriametrics.yaml"])["experiment_params"] + raw = yaml.safe_load(files["clickhouse.yaml"]) + mv = yaml.safe_load(files["clickhouse_mv.yaml"]) + + self.assertEqual( + [m["mode"] for m in e2e["experiment"]], ["sketchdb", "baseline"] + ) + self.assertEqual([m["mode"] for m in vm["experiment"]], ["baseline"]) + self.assertEqual(vm["monitoring"]["tool"], "victoriametrics") + self.assertEqual(raw["experiment"][0]["mode"], sbw.BASELINE) + self.assertEqual(mv["experiment"][0]["mode"], sbw.BASELINE_MV) + for config in (raw, mv): + self.assertEqual(config["dataset"]["clickhouse_image_tag"], "23.8") + self.assertIn( + "prebuilt_output_dir", yaml.safe_load(files["e2e.yaml"])["controller"] + ) + + feeders = [ + e2e["exporters"]["exporter_list"]["paced_feeder"], + vm["exporters"]["exporter_list"]["paced_feeder"], + raw["dataset"]["paced_feeder"], + mv["dataset"]["paced_feeder"], + ] + for feeder in feeders: + self.assertEqual( + (feeder["workload_table"], feeder["seed"], feeder["metrics"]), + (str((PILOT_DIR / "eval_table.json").resolve()), 862, ["data"]), + ) + # The longest lookback (20 min) plus the freshness delay, then 10 min. + schedules = [ + (g["repetition_delay_ms"], g["client_options"]) + for config in (e2e, vm, raw, mv) + for g in config["query_groups"] + ] + self.assertEqual({ms for ms, _ in schedules}, {1_000, 20_000, 30_000}) + for ms, options in schedules: + self.assertEqual(options["starting_delay"], 1210) + self.assertEqual(options["query_time_offset"], 10) + self.assertEqual(options["repetitions"], 600_000 // ms) + self.assertTrue(options["align_to_interval"]) + + pilot_sql = [ + q["clickhouse_sql"] + for q in workload["queries"] + if q["support"] == "common-promql" + ] + raw_sql = "".join(files[f"interval_{t}s_baseline.sql"] for t in (1, 20, 30)) + self.assertTrue(all(sql in raw_sql for sql in pilot_sql)) + + +class PilotConfigsPreflightTest(unittest.TestCase): + """Each config through Hydra and the checks its runner makes before it + starts any service, from a copy of the config directory.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.cwd = os.getcwd() + shutil.copytree( + Path(artifacts.EXPERIMENTS_DIR) / "config", Path(self.tmp.name) / "config" + ) + workload = workloads.generate_workloads(seed=862)[0] + for path, content in artifacts.pilot_configs(workload, PILOT_DIR).items(): + (Path(self.tmp.name) / path).parent.mkdir(parents=True, exist_ok=True) + (Path(self.tmp.name) / path).write_text(content) + # Configs name their SQL relative to the runners' directory. + os.chdir(self.tmp.name) + + def tearDown(self): + os.chdir(self.cwd) + self.tmp.cleanup() + + def compose(self, name): + with initialize_config_dir( + version_base=None, config_dir=os.path.join(self.tmp.name, "config") + ): + return compose( + "config", + overrides=[f"experiment_type=random_workload/workload-01/{name}"], + ) + + def test_e2e_and_victoriametrics_configs_validate(self): + for name, modes in ( + ("e2e", ["sketchdb", "baseline"]), + ("victoriametrics", ["baseline"]), + ): + with self.subTest(config=name): + cfg = self.compose(name) + params = cfg.experiment_params + experiment_config.validate_experiment_config(params) + exporter_config, reason = read_exporter_config(params) + self.assertIsNotNone(exporter_config, reason) + check_paced_feeder_config( + OmegaConf.to_container(exporter_config.exporter_list.paced_feeder), + modes=OmegaConf.to_container(params.experiment), + parallelism=cfg.streaming.parallelism, + min_feed_secs=cfg.flow.steady_state_wait + + experiment_config.get_minimum_experiment_running_time(params), + forward_unsupported_queries=cfg.streaming.forward_unsupported_queries, + auto_discover_labels=cfg.controller.auto_discover_labels, + ) + got, _ = experiment_config.generate_controller_client_configs( + params, os.path.join(self.tmp.name, name) + ) + self.assertEqual(got, modes) + + def test_clickhouse_configs_validate(self): + for name, mode in ( + ("clickhouse", sbw.BASELINE), + ("clickhouse_mv", sbw.BASELINE_MV), + ): + with self.subTest(config=name): + params = self.compose(name).experiment_params + experiment_config.validate_experiment_config(params) + out = os.path.join(self.tmp.name, name) + servers = {s.name: s.url for s in params.servers} + modes = experiment_config.generate_clickhouse_client_configs( + query_groups=params.query_groups, + local_experiment_dir=out, + mode_server_urls={mode: servers["clickhouse"]}, + clickhouse_database="default", + ) + self.assertEqual(modes, [mode]) + with open( + os.path.join(out, "controller_client_configs", f"{mode}.yaml") + ) as f: + client_config = yaml.safe_load(f) + align = _check_clickhouse_paced_feeder( + OmegaConf.to_container(params.dataset.paced_feeder), + modes, + client_config, + experiment_config.get_minimum_experiment_running_time(params), + ) + self.assertEqual(align, 60) + + +@unittest.skipUnless(CLICKHOUSE, "clickhouse binary not found") +class PilotMvMatchesRawTest(unittest.TestCase): + """The pilot's MV SQL answers what its raw SQL does, in clickhouse-local, + over 12 series sampled every 100 ms for 22 min.""" + + def test_mv_sql_matches_raw_sql(self): + _, files = pilot() + start_s, end_s = 1767225600, 1767225600 + 22 * 60 + rng = random.Random(3) + series = [ + (f"r{r}", f"s{r}{s}", f"j{r}{s}", f"l{r}{s}{i}") + for r in range(2) + for s in range(2) + for i in range(3) + ] + with tempfile.TemporaryDirectory() as tmp: + csv = os.path.join(tmp, "data.csv") + with open(csv, "w") as f: + for t in range(start_s * 1000, end_s * 1000, 100): + for labels in series: + f.write(f"{t},{','.join(labels)},{rng.paretovariate(2.0)!r}\n") + script = [ + files["init_mv.sql"], + "INSERT INTO data SELECT fromUnixTimestamp64Milli(ts_ms), region, " + f"service, job, label_0, value FROM file('{csv}', 'CSV', " + "'ts_ms Int64, region String, service String, job String, " + "label_0 String, value Float64');\n", + ] + for interval in (1, 20, 30): + for mode in (sbw.BASELINE, sbw.BASELINE_MV): + text = files[f"interval_{interval}s_{mode}.sql"] + for i, sql in enumerate(s for s in text.split(";\n") if s.strip()): + # Eval times past the 20 min warm-up, on every interval. + for end in (end_s - 60, end_s): + sql_at = sql.replace(sbw.END, str(end)) + script.append( + f"SELECT '{mode}/{interval}/{i}/{end}' AS qid, * " + f"FROM ({sql_at}) FORMAT JSONEachRow;\n" + ) + path = os.path.join(tmp, "script.sql") + with open(path, "w") as f: + f.write("".join(script)) + out = subprocess.run( + [CLICKHOUSE, "local", "--queries-file", path], + check=True, + capture_output=True, + text=True, + ).stdout + got = {} + for line in out.splitlines(): + row = json.loads(line) + mode, qid = row.pop("qid").split("/", 1) + value = row.pop("value") + got.setdefault(qid, {}).setdefault(mode, {})[tuple(row.items())] = value + self.assertEqual(len(got), 7 * 2) + for qid, by_mode in got.items(): + raw, mv = by_mode[sbw.BASELINE], by_mode[sbw.BASELINE_MV] + self.assertTrue(raw, qid) + self.assertEqual(sorted(raw), sorted(mv), qid) + for key, value in raw.items(): + self.assertTrue(math.isclose(mv[key], value, rel_tol=1e-9), qid) + if __name__ == "__main__": unittest.main() diff --git a/asap-tools/experiments/tests/test_random_workloads.py b/asap-tools/experiments/tests/test_random_workloads.py index 1cbad9f7..00a8bc97 100644 --- a/asap-tools/experiments/tests/test_random_workloads.py +++ b/asap-tools/experiments/tests/test_random_workloads.py @@ -44,6 +44,20 @@ def test_manifest_declares_each_querys_rendering_and_output_cardinality(self): else: self.assertIsNone(query["sql"]) self.assertIn(" by (", query["promql"]) + if query["support"] == "common-promql": + self.assertIsNotNone(query["clickhouse_sql"]) + + def test_grouped_quantiles_use_average_of_per_series_quantiles(self): + # The SQL each Query renders is checked against Prometheus in + # test_sketch_bench_workloads. + for query in workloads.generate_workloads(seed=4)[0]["queries"]: + rqe = query["rqes"][0] + if rqe["capability"] == "quantile": + self.assertIn("avg by", query["promql"]) + q = workloads.rqe_query(rqe) + self.assertEqual((q.series_fn, q.agg), ("quantile", "avg")) + self.assertEqual(q.grouping, tuple(query["grouping"])) + self.assertEqual(q.window_secs, 100) def test_visible_query_outputs_stay_bounded(self): for workload in workloads.generate_workloads(seed=4): @@ -53,6 +67,31 @@ def test_visible_query_outputs_stay_bounded(self): self.assertEqual(query["output_series_upper_bound"], expected) self.assertLessEqual(query["output_series_upper_bound"], 200) + def test_quantiles_cover_a_thousand_samples_per_series(self): + # sketch-bench's accuracy curves start at N = 1,000; below it no + # saved plan can serve the quantile. + for workload in workloads.generate_workloads(seed=4): + for query in workload["queries"]: + if query["rqes"][0]["capability"] == "quantile": + samples = ( + query["lookback_ms"] + // workload["data_facts"]["scrape_interval_ms"] + ) + self.assertEqual(samples, 1_000) + self.assertIn("data[100s]", query["promql"]) + self.assertEqual(query["interval_ms"], workloads.SECOND_MS) + + def test_manifest_declares_deterministic_feeder_facts(self): + facts = workloads.generate_workloads(seed=4)[0]["data_facts"] + + self.assertEqual(facts["scrape_interval_ms"], 100) + self.assertEqual(facts["series"]["leaf_count"], 10_000) + self.assertEqual(facts["series"]["ranking"]["exponent"], 1.1) + self.assertEqual(facts["metrics"]["data"]["pareto_shape"], 2.0) + self.assertEqual(facts["metrics"]["data_total"]["resets"], "none") + self.assertEqual(facts["events"]["user_id_universe_size"], 10_000) + self.assertEqual(facts["events"]["user_ids_per_leaf"], 1_000) + def test_workloads_do_not_repeat_an_atomic_rqe(self): for workload in workloads.generate_workloads(seed=1): keys = [] diff --git a/asap-tools/experiments/tests/test_sketch_bench_workloads.py b/asap-tools/experiments/tests/test_sketch_bench_workloads.py index 8b3540b3..1330a381 100644 --- a/asap-tools/experiments/tests/test_sketch_bench_workloads.py +++ b/asap-tools/experiments/tests/test_sketch_bench_workloads.py @@ -653,6 +653,49 @@ def q_ratio(t, gauge): lambda t, g, c: q_ratio(t, g), ), ] + + +def topk_by(k: int, vec: Vector, idx: List[int]) -> Vector: + out: Vector = {} + for group in {tuple(s[i] for i in idx) for s in vec}: + members = {s: v for s, v in vec.items() if tuple(s[i] for i in idx) == group} + out.update(topk(k, members)) + return out + + +ALL_LABELS = ("job", "label_0") +# Shapes the random workloads build from their RQEs, not from PromQL: +# (name, Query, reference over (gauge, counter) at eval time t ms). +RQE_CASES = [ + ( + "avg by (job) (quantile_over_time(0.9, data[2m]))", + sbw.Query("data", 60, 120, ("job",), "quantile", "avg", "0.9"), + lambda t, g, c: agg_by( + range_fn(g, t, W2, lambda w: prom_quantile(0.9, values(w))), + JOB, + lambda vs: sum(vs) / len(vs), + ), + ), + ( + "topk by (job) (2, sum_over_time(data[2m]))", + sbw.Query("data", 60, 120, ALL_LABELS, None, "sum", topk=2, topk_by=("job",)), + lambda t, g, c: topk_by(2, range_fn(g, t, W2, lambda w: sum(values(w))), JOB), + ), + ( + "topk by (job) (2, increase(data_total[5m]))", + sbw.Query( + "data_total", + 60, + 300, + ALL_LABELS, + "increase", + None, + topk=2, + topk_by=("job",), + ), + lambda t, g, c: topk_by(2, rate(c, t, W5, False), JOB), + ), +] EVAL_OFFSETS_SECS = { # Before data, data start (the only sample sits on the window's end), # mid-run, around series 2's gap and series 1's end, and after all data. @@ -814,6 +857,10 @@ def all_cases(gauge, counter, records) -> List[Tuple[str, "sbw.Query", Callable] (p, sbw.parse_promql(p, t, LABELS), lambda e, ref=ref: ref(e, gauge, counter)) for t, p, ref in CASES ] + cases += [ + (name, q, lambda e, ref=ref: ref(e, gauge, counter)) + for name, q, ref in RQE_CASES + ] for entry in MULTIGROUP: q = sbw.query_from_entry(entry, METRICS, LABELS) w_ms = q.window_secs * 1000 @@ -954,9 +1001,10 @@ def test_raw_sql_matches_prometheus_reference(self): def test_mv_sql_matches_raw_sql(self): for i, (name, q, _) in enumerate(self.cases): - # Only sums are re-associated (per bucket, then across buckets and - # groups); every other statistic merges to the identical value. - rel = 1e-9 if q.agg == "sum" else 0.0 + # Only sums and averages are re-associated (per bucket, then across + # buckets and groups); every other statistic merges to the + # identical value. + rel = 1e-9 if q.agg in ("sum", "avg") else 0.0 for offset in EVAL_OFFSETS_SECS[q.interval_secs]: with self.subTest(query=name, offset=offset): raw = self.got[f"raw/{i}@{offset}"]