Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(planner): improve histogram cardinality estimation #17200

Merged
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions src/common/storage/src/statistics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,14 @@ impl Datum {
matches!(self, Datum::Bytes(_))
}

pub fn to_float(self) -> Self {
match self {
Datum::Int(v) => Datum::Float(F64::from(v as f64)),
Datum::UInt(v) => Datum::Float(F64::from(v as f64)),
_ => self,
}
}

pub fn to_double(&self) -> Result<f64> {
match self {
Datum::Bool(v) => Ok(*v as u8 as f64),
Expand Down
7 changes: 5 additions & 2 deletions src/query/sql/src/planner/optimizer/property/histogram.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,15 @@

use std::cmp::Ordering;

use databend_common_base::base::OrderedFloat;
use databend_common_exception::Result;
use databend_common_expression::arithmetics_type::ResultTypeOfUnary;
use databend_common_storage::Datum;
use databend_common_storage::Histogram;
use databend_common_storage::HistogramBucket;

pub type F64 = OrderedFloat<f64>;

/// Construct a histogram from NDV and total number of rows.
///
/// # Arguments
Expand Down Expand Up @@ -59,7 +62,7 @@ pub fn histogram_from_ndv(
}

let (min, max) = match bound {
Some((min, max)) => (min, max),
Some((min, max)) => (min.to_float(), max.to_float()),
None => {
return Err(format!(
"Must have min and max value when NDV is greater than 0, got NDV: {}",
Expand Down Expand Up @@ -182,7 +185,7 @@ impl SampleSet for UniformSampleSet {

(Datum::Float(min), Datum::Float(max)) => {
let min = *min;
let max = *max;
let max = (*max).checked_add(F64::from(1.0)).ok_or("overflowed")?;
// TODO(xudong): better histogram computation.
let bucket_range = max.checked_sub(min).ok_or("overflowed")? / num_buckets as f64;
let upper_bound = min + bucket_range * bucket_index as f64;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,22 +63,22 @@ Memo
│ └── #0 Join [#0, #3]
├── Group #5
│ ├── Best properties
│ │ ├── { dist: Any }: expr: #0, cost: 4419.000, children: [{ dist: Any }]
│ │ └── { dist: Serial }: expr: #1, cost: 7569.000, children: [{ dist: Any }]
│ │ ├── { dist: Any }: expr: #0, cost: 4420.000, children: [{ dist: Any }]
│ │ └── { dist: Serial }: expr: #1, cost: 7920.000, children: [{ dist: Any }]
│ ├── #0 EvalScalar [#4]
│ └── #1 Exchange: (Merge) [#5]
├── Group #6
│ ├── Best properties
│ │ └── { dist: Serial }: expr: #0, cost: 7614.000, children: [{ dist: Serial }]
│ │ └── { dist: Serial }: expr: #0, cost: 7970.000, children: [{ dist: Serial }]
│ ├── #0 Aggregate [#5]
│ └── #1 Exchange: (Merge) [#6]
├── Group #7
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 7619.000, children: [{ dist: Serial }]
│ │ └── { dist: Any }: expr: #0, cost: 7975.000, children: [{ dist: Serial }]
│ └── #0 Aggregate [#6]
└── Group #8
├── Best properties
│ └── { dist: Serial }: expr: #0, cost: 7620.000, children: [{ dist: Any }]
│ └── { dist: Serial }: expr: #0, cost: 7976.000, children: [{ dist: Any }]
└── #0 EvalScalar [#7]

query T
Expand Down Expand Up @@ -126,22 +126,22 @@ Memo
│ └── #0 Join [#0, #3]
├── Group #5
│ ├── Best properties
│ │ ├── { dist: Any }: expr: #0, cost: 4419.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_10.a (#0)::Int32 NULL) }: expr: #1, cost: 4878.000, children: [{ dist: Any }]
│ │ ├── { dist: Any }: expr: #0, cost: 4420.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_10.a (#0)::Int32 NULL) }: expr: #1, cost: 4930.000, children: [{ dist: Any }]
│ ├── #0 EvalScalar [#4]
│ └── #1 Exchange: (Hash(t_10.a (#0)::Int32 NULL)) [#5]
├── Group #6
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 4923.000, children: [{ dist: Hash(t_10.a (#0)::Int32 NULL) }]
│ │ └── { dist: Any }: expr: #0, cost: 4980.000, children: [{ dist: Hash(t_10.a (#0)::Int32 NULL) }]
│ └── #0 Aggregate [#5]
├── Group #7
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 4968.000, children: [{ dist: Any }]
│ │ └── { dist: Any }: expr: #0, cost: 5030.000, children: [{ dist: Any }]
│ └── #0 Aggregate [#6]
├── Group #8
│ ├── Best properties
│ │ ├── { dist: Any }: expr: #0, cost: 4977.000, children: [{ dist: Any }]
│ │ └── { dist: Serial }: expr: #4, cost: 8127.000, children: [{ dist: Any }]
│ │ ├── { dist: Any }: expr: #0, cost: 5040.000, children: [{ dist: Any }]
│ │ └── { dist: Serial }: expr: #4, cost: 8540.000, children: [{ dist: Any }]
│ ├── #0 EvalScalar [#7]
│ ├── #1 EvalScalar [#14]
│ ├── #2 EvalScalar [#20]
Expand All @@ -166,16 +166,16 @@ Memo
├── Group #12
│ ├── Best properties
│ │ ├── { dist: Any }: expr: #0, cost: 66410.000, children: [{ dist: Any }, { dist: Broadcast }]
│ │ └── { dist: Hash(t_10.a (#0)::Int32 NULL) }: expr: #1, cost: 66869.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_10.a (#0)::Int32 NULL) }: expr: #1, cost: 66920.000, children: [{ dist: Any }]
│ ├── #0 Join [#11, #3]
│ └── #1 Exchange: (Hash(t_10.a (#0)::Int32 NULL)) [#12]
├── Group #13
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 66914.000, children: [{ dist: Hash(t_10.a (#0)::Int32 NULL) }]
│ │ └── { dist: Any }: expr: #0, cost: 66970.000, children: [{ dist: Hash(t_10.a (#0)::Int32 NULL) }]
│ └── #0 Aggregate [#12]
├── Group #14
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 66959.000, children: [{ dist: Any }]
│ │ └── { dist: Any }: expr: #0, cost: 67020.000, children: [{ dist: Any }]
│ └── #0 Aggregate [#13]
├── Group #15
│ ├── Best properties
Expand All @@ -197,35 +197,35 @@ Memo
│ └── #0 Join [#0, #16]
├── Group #18
│ ├── Best properties
│ │ ├── { dist: Any }: expr: #0, cost: 5029.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_10.a (#0)::Int32 NULL) }: expr: #1, cost: 5488.000, children: [{ dist: Any }]
│ │ ├── { dist: Any }: expr: #0, cost: 5030.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_10.a (#0)::Int32 NULL) }: expr: #1, cost: 5540.000, children: [{ dist: Any }]
│ ├── #0 EvalScalar [#17]
│ └── #1 Exchange: (Hash(t_10.a (#0)::Int32 NULL)) [#18]
├── Group #19
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 5533.000, children: [{ dist: Hash(t_10.a (#0)::Int32 NULL) }]
│ │ └── { dist: Any }: expr: #0, cost: 5590.000, children: [{ dist: Hash(t_10.a (#0)::Int32 NULL) }]
│ └── #0 Aggregate [#18]
├── Group #20
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 5578.000, children: [{ dist: Any }]
│ │ └── { dist: Any }: expr: #0, cost: 5640.000, children: [{ dist: Any }]
│ └── #0 Aggregate [#19]
├── Group #21
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 67020.000, children: [{ dist: Any }, { dist: Broadcast }]
│ └── #0 Join [#11, #16]
├── Group #22
│ ├── Best properties
│ │ ├── { dist: Any }: expr: #0, cost: 67029.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_10.a (#0)::Int32 NULL) }: expr: #1, cost: 67488.000, children: [{ dist: Any }]
│ │ ├── { dist: Any }: expr: #0, cost: 67030.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_10.a (#0)::Int32 NULL) }: expr: #1, cost: 67540.000, children: [{ dist: Any }]
│ ├── #0 EvalScalar [#21]
│ └── #1 Exchange: (Hash(t_10.a (#0)::Int32 NULL)) [#22]
├── Group #23
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 67533.000, children: [{ dist: Hash(t_10.a (#0)::Int32 NULL) }]
│ │ └── { dist: Any }: expr: #0, cost: 67590.000, children: [{ dist: Hash(t_10.a (#0)::Int32 NULL) }]
│ └── #0 Aggregate [#22]
└── Group #24
├── Best properties
│ └── { dist: Any }: expr: #0, cost: 67578.000, children: [{ dist: Any }]
│ └── { dist: Any }: expr: #0, cost: 67640.000, children: [{ dist: Any }]
└── #0 Aggregate [#23]


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@ Memo
│ └── #0 Join [#0, #3]
└── Group #5
├── Best properties
│ ├── { dist: Any }: expr: #0, cost: 4419.000, children: [{ dist: Any }]
│ └── { dist: Serial }: expr: #1, cost: 7569.000, children: [{ dist: Any }]
│ ├── { dist: Any }: expr: #0, cost: 4420.000, children: [{ dist: Any }]
│ └── { dist: Serial }: expr: #1, cost: 7920.000, children: [{ dist: Any }]
├── #0 EvalScalar [#4]
└── #1 Exchange: (Merge) [#5]

Expand Down Expand Up @@ -192,8 +192,8 @@ Memo
│ └── #0 Join [#2, #3]
└── Group #5
├── Best properties
│ ├── { dist: Any }: expr: #0, cost: 112911.000, children: [{ dist: Any }]
│ └── { dist: Serial }: expr: #1, cost: 494761.000, children: [{ dist: Any }]
│ ├── { dist: Any }: expr: #0, cost: 112910.000, children: [{ dist: Any }]
│ └── { dist: Serial }: expr: #1, cost: 494410.000, children: [{ dist: Any }]
├── #0 EvalScalar [#4]
└── #1 Exchange: (Merge) [#5]

Expand Down
38 changes: 19 additions & 19 deletions tests/sqllogictests/suites/mode/cluster/memo/mix_property.test
Original file line number Diff line number Diff line change
Expand Up @@ -66,34 +66,34 @@ Memo
│ └── #0 Join [#0, #3]
├── Group #5
│ ├── Best properties
│ │ ├── { dist: Any }: expr: #0, cost: 4419.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_100.a (#1)::Int32 NULL) }: expr: #1, cost: 4878.000, children: [{ dist: Any }]
│ │ ├── { dist: Any }: expr: #0, cost: 4420.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_100.a (#1)::Int32 NULL) }: expr: #1, cost: 4930.000, children: [{ dist: Any }]
│ ├── #0 EvalScalar [#4]
│ └── #1 Exchange: (Hash(t_100.a (#1)::Int32 NULL)) [#5]
├── Group #6
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 4923.000, children: [{ dist: Hash(t_100.a (#1)::Int32 NULL) }]
│ │ └── { dist: Any }: expr: #0, cost: 4980.000, children: [{ dist: Hash(t_100.a (#1)::Int32 NULL) }]
│ └── #0 Aggregate [#5]
├── Group #7
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 4968.000, children: [{ dist: Any }]
│ │ └── { dist: Any }: expr: #0, cost: 5030.000, children: [{ dist: Any }]
│ └── #0 Aggregate [#6]
├── Group #8
│ ├── Best properties
│ │ ├── { dist: Any }: expr: #0, cost: 4977.000, children: [{ dist: Any }]
│ │ └── { dist: Serial }: expr: #4, cost: 8127.000, children: [{ dist: Any }]
│ │ ├── { dist: Any }: expr: #0, cost: 5040.000, children: [{ dist: Any }]
│ │ └── { dist: Serial }: expr: #4, cost: 8540.000, children: [{ dist: Any }]
│ ├── #0 EvalScalar [#7]
│ ├── #1 EvalScalar [#16]
│ ├── #2 EvalScalar [#22]
│ ├── #3 EvalScalar [#26]
│ └── #4 Exchange: (Merge) [#8]
├── Group #9
│ ├── Best properties
│ │ └── { dist: Serial }: expr: #0, cost: 8136.000, children: [{ dist: Serial }]
│ │ └── { dist: Serial }: expr: #0, cost: 8550.000, children: [{ dist: Serial }]
│ └── #0 Sort [#8]
├── Group #10
│ ├── Best properties
│ │ └── { dist: Serial }: expr: #0, cost: 8145.000, children: [{ dist: Serial }]
│ │ └── { dist: Serial }: expr: #0, cost: 8560.000, children: [{ dist: Serial }]
│ └── #0 Limit [#9]
├── Group #11
│ ├── Best properties
Expand All @@ -114,16 +114,16 @@ Memo
├── Group #14
│ ├── Best properties
│ │ ├── { dist: Any }: expr: #0, cost: 66410.000, children: [{ dist: Any }, { dist: Broadcast }]
│ │ └── { dist: Hash(t_100.a (#1)::Int32 NULL) }: expr: #1, cost: 66869.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_100.a (#1)::Int32 NULL) }: expr: #1, cost: 66920.000, children: [{ dist: Any }]
│ ├── #0 Join [#13, #3]
│ └── #1 Exchange: (Hash(t_100.a (#1)::Int32 NULL)) [#14]
├── Group #15
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 66914.000, children: [{ dist: Hash(t_100.a (#1)::Int32 NULL) }]
│ │ └── { dist: Any }: expr: #0, cost: 66970.000, children: [{ dist: Hash(t_100.a (#1)::Int32 NULL) }]
│ └── #0 Aggregate [#14]
├── Group #16
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 66959.000, children: [{ dist: Any }]
│ │ └── { dist: Any }: expr: #0, cost: 67020.000, children: [{ dist: Any }]
│ └── #0 Aggregate [#15]
├── Group #17
│ ├── Best properties
Expand All @@ -145,35 +145,35 @@ Memo
│ └── #0 Join [#0, #18]
├── Group #20
│ ├── Best properties
│ │ ├── { dist: Any }: expr: #0, cost: 9129.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_100.a (#1)::Int32 NULL) }: expr: #1, cost: 9588.000, children: [{ dist: Any }]
│ │ ├── { dist: Any }: expr: #0, cost: 9130.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_100.a (#1)::Int32 NULL) }: expr: #1, cost: 9640.000, children: [{ dist: Any }]
│ ├── #0 EvalScalar [#19]
│ └── #1 Exchange: (Hash(t_100.a (#1)::Int32 NULL)) [#20]
├── Group #21
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 9633.000, children: [{ dist: Hash(t_100.a (#1)::Int32 NULL) }]
│ │ └── { dist: Any }: expr: #0, cost: 9690.000, children: [{ dist: Hash(t_100.a (#1)::Int32 NULL) }]
│ └── #0 Aggregate [#20]
├── Group #22
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 9678.000, children: [{ dist: Any }]
│ │ └── { dist: Any }: expr: #0, cost: 9740.000, children: [{ dist: Any }]
│ └── #0 Aggregate [#21]
├── Group #23
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 71120.000, children: [{ dist: Any }, { dist: Broadcast }]
│ └── #0 Join [#13, #18]
├── Group #24
│ ├── Best properties
│ │ ├── { dist: Any }: expr: #0, cost: 71129.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_100.a (#1)::Int32 NULL) }: expr: #1, cost: 71588.000, children: [{ dist: Any }]
│ │ ├── { dist: Any }: expr: #0, cost: 71130.000, children: [{ dist: Any }]
│ │ └── { dist: Hash(t_100.a (#1)::Int32 NULL) }: expr: #1, cost: 71640.000, children: [{ dist: Any }]
│ ├── #0 EvalScalar [#23]
│ └── #1 Exchange: (Hash(t_100.a (#1)::Int32 NULL)) [#24]
├── Group #25
│ ├── Best properties
│ │ └── { dist: Any }: expr: #0, cost: 71633.000, children: [{ dist: Hash(t_100.a (#1)::Int32 NULL) }]
│ │ └── { dist: Any }: expr: #0, cost: 71690.000, children: [{ dist: Hash(t_100.a (#1)::Int32 NULL) }]
│ └── #0 Aggregate [#24]
└── Group #26
├── Best properties
│ └── { dist: Any }: expr: #0, cost: 71678.000, children: [{ dist: Any }]
│ └── { dist: Any }: expr: #0, cost: 71740.000, children: [{ dist: Any }]
└── #0 Aggregate [#25]


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,7 @@ explain select * from t left join t t1 on t.a = t1.a where t1.a <= 1 or t.a < 1
Filter
├── output columns: [t.a (#0), t1.a (#1)]
├── filters: [is_true((t1.a (#1) <= 1 OR t.a (#0) < 1))]
├── estimated rows: 3.09
├── estimated rows: 2.80
└── HashJoin
├── output columns: [t.a (#0), t1.a (#1)]
├── join type: LEFT OUTER
Expand Down Expand Up @@ -574,7 +574,7 @@ explain select * from t left join t t1 on t.a = t1.a where t1.a <= 1 or (t.a > 1
Filter
├── output columns: [t.a (#0), t1.a (#1)]
├── filters: [is_true((t1.a (#1) <= 1 OR (t.a (#0) > 1 AND t1.a (#1) > 1)))]
├── estimated rows: 6.95
├── estimated rows: 7.65
└── HashJoin
├── output columns: [t.a (#0), t1.a (#1)]
├── join type: INNER
Expand Down Expand Up @@ -617,7 +617,7 @@ explain select * from t left join t t1 on t.a = t1.a where t1.a <= 1 or (t.a > 1
Filter
├── output columns: [t.a (#0), t1.a (#1)]
├── filters: [is_true((t1.a (#1) <= 1 OR (t.a (#0) > 1 AND t.a (#0) < 2)))]
├── estimated rows: 3.95
├── estimated rows: 3.60
└── HashJoin
├── output columns: [t.a (#0), t1.a (#1)]
├── join type: LEFT OUTER
Expand Down
32 changes: 16 additions & 16 deletions tests/sqllogictests/suites/mode/standalone/explain/join.test
Original file line number Diff line number Diff line change
Expand Up @@ -272,39 +272,39 @@ query T
explain select * from onecolumn as a left join twocolumn as b on a.x = b.x where b.x > 44 or b.x < 43
----
HashJoin
├── output columns: [a.x (#0), b.y (#2), b.x (#1)]
├── output columns: [b.x (#1), b.y (#2), a.x (#0)]
├── join type: INNER
├── build keys: [b.x (#1)]
├── probe keys: [a.x (#0)]
├── build keys: [a.x (#0)]
├── probe keys: [b.x (#1)]
├── filters: []
├── estimated rows: 0.89
├── estimated rows: 1.33
├── Filter(Build)
│ ├── output columns: [b.x (#1), b.y (#2)]
│ ├── filters: [is_true((b.x (#1) > 44 OR b.x (#1) < 43))]
│ ├── output columns: [a.x (#0)]
│ ├── filters: [is_true((a.x (#0) > 44 OR a.x (#0) < 43))]
│ ├── estimated rows: 1.33
│ └── TableScan
│ ├── table: default.default.twocolumn
│ ├── output columns: [x (#1), y (#2)]
│ ├── table: default.default.onecolumn
│ ├── output columns: [x (#0)]
│ ├── read rows: 4
│ ├── read size: < 1 KiB
│ ├── partitions total: 1
│ ├── partitions scanned: 1
│ ├── pruning stats: [segments: <range pruning: 1 to 1>, blocks: <range pruning: 1 to 1>]
│ ├── push downs: [filters: [is_true((twocolumn.x (#1) > 44 OR twocolumn.x (#1) < 43))], limit: NONE]
│ ├── push downs: [filters: [is_true((onecolumn.x (#0) > 44 OR onecolumn.x (#0) < 43))], limit: NONE]
│ └── estimated rows: 4.00
└── Filter(Probe)
├── output columns: [a.x (#0)]
├── filters: [is_true((a.x (#0) > 44 OR a.x (#0) < 43))]
├── estimated rows: 1.33
├── output columns: [b.x (#1), b.y (#2)]
├── filters: [is_true((b.x (#1) > 44 OR b.x (#1) < 43))]
├── estimated rows: 2.00
└── TableScan
├── table: default.default.onecolumn
├── output columns: [x (#0)]
├── table: default.default.twocolumn
├── output columns: [x (#1), y (#2)]
├── read rows: 4
├── read size: < 1 KiB
├── partitions total: 1
├── partitions scanned: 1
├── pruning stats: [segments: <range pruning: 1 to 1>, blocks: <range pruning: 1 to 1>]
├── push downs: [filters: [is_true((onecolumn.x (#0) > 44 OR onecolumn.x (#0) < 43))], limit: NONE]
├── push downs: [filters: [is_true((twocolumn.x (#1) > 44 OR twocolumn.x (#1) < 43))], limit: NONE]
└── estimated rows: 4.00

query T
Expand Down Expand Up @@ -354,7 +354,7 @@ explain select * from onecolumn as a left join twocolumn as b on a.x = b.x where
Filter
├── output columns: [a.x (#0), b.x (#1), b.y (#2)]
├── filters: [is_true((b.x (#1) > 44 OR a.x (#0) < 43))]
├── estimated rows: 2.00
├── estimated rows: 1.33
└── HashJoin
├── output columns: [a.x (#0), b.x (#1), b.y (#2)]
├── join type: LEFT OUTER
Expand Down
Loading
Loading