From d0a20d91cb61ca954aee283667fa9039117a37f3 Mon Sep 17 00:00:00 2001 From: JackTan25 Date: Thu, 2 Nov 2023 16:30:44 +0800 Subject: [PATCH] improve --- .../processor_deduplicate_row_number.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/query/service/src/pipelines/processors/transforms/processor_deduplicate_row_number.rs b/src/query/service/src/pipelines/processors/transforms/processor_deduplicate_row_number.rs index f5fa3e0b74d2c..5b113e7160f64 100644 --- a/src/query/service/src/pipelines/processors/transforms/processor_deduplicate_row_number.rs +++ b/src/query/service/src/pipelines/processors/transforms/processor_deduplicate_row_number.rs @@ -67,21 +67,20 @@ impl DeduplicateRowNumber { } let row_number_vec = get_row_number(&data_block, 0); - let length = row_number_vec.len(); - let row_number_set: HashSet = row_number_vec.into_iter().collect(); - assert_eq!(row_number_set.len(), length); if !self.accepted_data { - self.unique_row_number = row_number_set; + self.unique_row_number = row_number_vec.into_iter().collect(); self.accepted_data = true; return Ok(()); } - self.unique_row_number = self - .unique_row_number - .intersection(&row_number_set) - .cloned() - .collect(); + let mut new_set = HashSet::with_capacity(self.unique_row_number.len()); + for number in row_number_vec { + if self.unique_row_number.contains(&number) { + new_set.insert(number); + } + } + self.unique_row_number = new_set; Ok(()) }