refactor create agg index

databendlabs · Dec 19, 2023 · 5294976 · 5294976
1 parent 37e04b9
commit 5294976
Show file tree

Hide file tree

Showing 21 changed files with 341 additions and 91 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/src/meta/api/src/schema_api_test_suite.rs b/src/meta/api/src/schema_api_test_suite.rs
@@ -3528,8 +3528,11 @@ impl SchemaApiTestSuite {
                 created_on: Utc::now(),
                 dropped_on: None,
                 updated_on: None,
-                query: "select sum(number) from tb1".to_string(),
+                original_query: "select sum(number) from tb1".to_string(),
+                query: "select sum(number), tb1._block_name from tb1 group by tb1._block_name"
+                    .to_string(),
                 sync_creation: false,
+                user_defined_block_name: false,
             },
         };
 
@@ -5540,8 +5543,12 @@ impl SchemaApiTestSuite {
             created_on,
             dropped_on: None,
             updated_on: None,
-            query: "SELECT a, SUM(b) FROM tb1 WHERE a > 1 GROUP BY b".to_string(),
+            original_query: "SELECT a, SUM(b) FROM tb1 WHERE a > 1 GROUP BY b".to_string(),
+            query:
+                "SELECT a, SUM(b), tb1._block_name FROM tb1 WHERE a > 1 GROUP BY b, tb1._block_name"
+                    .to_string(),
             sync_creation: false,
+            user_defined_block_name: false,
         };
 
         let index_name_2 = "idx2";
@@ -5551,8 +5558,12 @@ impl SchemaApiTestSuite {
             created_on,
             dropped_on: None,
             updated_on: None,
-            query: "SELECT a, SUM(b) FROM tb1 WHERE b > 1 GROUP BY b".to_string(),
+            original_query: "SELECT a, SUM(b) FROM tb1 WHERE b > 1 GROUP BY b".to_string(),
+            query:
+                "SELECT a, SUM(b), tb1._block_name FROM tb1 WHERE b > 1 GROUP BY b, tb1._block_name"
+                    .to_string(),
             sync_creation: false,
+            user_defined_block_name: false,
         };
 
         let name_ident_1 = IndexNameIdent {

diff --git a/src/meta/app/src/schema/index.rs b/src/meta/app/src/schema/index.rs
@@ -101,10 +101,13 @@ pub struct IndexMeta {
     // if used in CreateIndexReq, `dropped_on` and `updated_on` MUST set to None.
     pub dropped_on: Option<DateTime<Utc>>,
     pub updated_on: Option<DateTime<Utc>>,
+    pub original_query: String,
     pub query: String,
     // if true, index will create after data written to databend,
     // no need execute refresh index manually.
     pub sync_creation: bool,
+    // if user already use the internal column name `_block_name` in their sql.
+    pub user_defined_block_name: bool,
 }
 
 impl Default for IndexMeta {
@@ -115,8 +118,10 @@ impl Default for IndexMeta {
             created_on: Utc::now(),
             dropped_on: None,
             updated_on: None,
+            original_query: "".to_string(),
             query: "".to_string(),
             sync_creation: false,
+            user_defined_block_name: false,
         }
     }
 }

diff --git a/src/meta/proto-conv/src/index_from_to_protobuf_impl.rs b/src/meta/proto-conv/src/index_from_to_protobuf_impl.rs
@@ -81,8 +81,10 @@ impl FromToProto for mt::IndexMeta {
                 Some(update_on) => Some(DateTime::<Utc>::from_pb(update_on)?),
                 None => None,
             },
+            original_query: p.original_query,
             query: p.query,
             sync_creation: p.sync_creation,
+            user_defined_block_name: p.user_defined_block_name,
         };
         Ok(v)
     }
@@ -102,8 +104,10 @@ impl FromToProto for mt::IndexMeta {
                 Some(update_on) => Some(update_on.to_pb()?),
                 None => None,
             },
+            original_query: self.original_query.clone(),
             query: self.query.clone(),
             sync_creation: self.sync_creation,
+            user_defined_block_name: self.user_defined_block_name,
         };
         Ok(p)
     }

diff --git a/src/meta/proto-conv/src/util.rs b/src/meta/proto-conv/src/util.rs
@@ -96,6 +96,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
     (64, "2023-11-16: Add: user.proto/NDJsonFileFormatParams add field `missing_field_as` and `null_field_as`", ),
     (65, "2023-11-16: Retype: use Datetime<Utc> instead of u64 to in lvt.time", ),
     (66, "2023-12-15: Add: stage.proto/StageInfo::created_on", ),
+    (67, "2023-12-19: Add: index.proto/IndexMeta add field `original_query` and `user_defined_block_name`")
     // Dear developer:
     //      If you're gonna add a new metadata version, you'll have to add a test for it.
     //      You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)

diff --git a/src/meta/proto-conv/tests/it/main.rs b/src/meta/proto-conv/tests/it/main.rs
@@ -70,3 +70,4 @@ mod v063_connection;
 mod v064_ndjson_format_params;
 mod v065_least_visible_time;
 mod v066_stage_create_on;
+mod v067_index_meta;
diff --git a/src/meta/proto-conv/tests/it/proto_conv.rs b/src/meta/proto-conv/tests/it/proto_conv.rs
@@ -224,8 +224,10 @@ fn new_index_meta() -> mt::IndexMeta {
         created_on: Utc.with_ymd_and_hms(2015, 3, 9, 20, 0, 9).unwrap(),
         dropped_on: None,
         updated_on: None,
-        query: "SELECT a, sum(b) FROM default.t1 WHERE a > 3 GROUP BY b".to_string(),
+        original_query: "SELECT a, sum(b) FROM default.t1 WHERE a > 3 GROUP BY b".to_string(),
+        query: "SELECT a, SUM(b), t1._block_name FROM default.t1 WHERE a > 3 GROUP BY b, t1._block_name".to_string(),
         sync_creation: false,
+        user_defined_block_name:false,
     }
 }
 
@@ -457,7 +459,7 @@ fn test_build_pb_buf() -> anyhow::Result<()> {
 
         let mut buf = vec![];
         databend_common_protos::prost::Message::encode(&p, &mut buf)?;
-        println!("index:{buf:?}");
+        println!("index meta:{buf:?}");
     }
 
     // TableCopiedFileInfo

diff --git a/src/meta/proto-conv/tests/it/v037_index_meta.rs b/src/meta/proto-conv/tests/it/v037_index_meta.rs
@@ -44,16 +44,19 @@ fn test_decode_v37_index() -> anyhow::Result<()> {
         let table_id = 7;
         let index_type = IndexType::AGGREGATING;
         let created_on = Utc.with_ymd_and_hms(2015, 3, 9, 20, 0, 9).unwrap();
+        let original_query = "".to_string();
         let query = "SELECT a, sum(b) FROM default.t1 WHERE a > 3 GROUP BY b".to_string();
 
         IndexMeta {
             table_id,
             index_type,
             created_on,
             dropped_on: None,
+            original_query,
             query,
             updated_on: None,
             sync_creation: false,
+            user_defined_block_name: false,
         }
     };
 

diff --git a/src/meta/proto-conv/tests/it/v046_index_meta.rs b/src/meta/proto-conv/tests/it/v046_index_meta.rs
@@ -44,16 +44,19 @@ fn test_decode_v46_index() -> anyhow::Result<()> {
         let table_id = 7;
         let index_type = IndexType::AGGREGATING;
         let created_on = Utc.with_ymd_and_hms(2015, 3, 9, 20, 0, 9).unwrap();
+        let original_query = "".to_string();
         let query = "SELECT a, sum(b) FROM default.t1 WHERE a > 3 GROUP BY b".to_string();
 
         IndexMeta {
             table_id,
             index_type,
             created_on,
             dropped_on: None,
+            original_query,
             query,
             updated_on: None,
             sync_creation: false,
+            user_defined_block_name: false,
         }
     };
 

diff --git a/src/meta/proto-conv/tests/it/v054_index_meta.rs b/src/meta/proto-conv/tests/it/v054_index_meta.rs
@@ -44,16 +44,19 @@ fn test_decode_v54_index() -> anyhow::Result<()> {
         let table_id = 7;
         let index_type = IndexType::AGGREGATING;
         let created_on = Utc.with_ymd_and_hms(2015, 3, 9, 20, 0, 9).unwrap();
+        let original_query = "".to_string();
         let query = "SELECT a, sum(b) FROM default.t1 WHERE a > 3 GROUP BY b".to_string();
 
         IndexMeta {
             table_id,
             index_type,
             created_on,
             dropped_on: None,
+            original_query,
             query,
             updated_on: None,
             sync_creation: false,
+            user_defined_block_name: false,
         }
     };
 

diff --git a/src/meta/proto-conv/tests/it/v067_index_meta.rs b/src/meta/proto-conv/tests/it/v067_index_meta.rs
@@ -0,0 +1,71 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use chrono::TimeZone;
+use chrono::Utc;
+use databend_common_meta_app::schema::IndexMeta;
+use databend_common_meta_app::schema::IndexType;
+use minitrace::func_name;
+
+use crate::common;
+
+// These bytes are built when a new version in introduced,
+// and are kept for backward compatibility test.
+//
+// *************************************************************
+// * These messages should never be updated,                   *
+// * only be added when a new version is added,                *
+// * or be removed when an old version is no longer supported. *
+// *************************************************************
+//
+// The message bytes are built from the output of `proto_conv::test_build_pb_buf()`
+#[test]
+fn test_decode_v67_index() -> anyhow::Result<()> {
+    let index_v067 = vec![
+        8, 7, 16, 1, 26, 23, 50, 48, 49, 53, 45, 48, 51, 45, 48, 57, 32, 50, 48, 58, 48, 48, 58,
+        48, 57, 32, 85, 84, 67, 42, 87, 83, 69, 76, 69, 67, 84, 32, 97, 44, 32, 83, 85, 77, 40, 98,
+        41, 44, 32, 116, 49, 46, 95, 98, 108, 111, 99, 107, 95, 110, 97, 109, 101, 32, 70, 82, 79,
+        77, 32, 100, 101, 102, 97, 117, 108, 116, 46, 116, 49, 32, 87, 72, 69, 82, 69, 32, 97, 32,
+        62, 32, 51, 32, 71, 82, 79, 85, 80, 32, 66, 89, 32, 98, 44, 32, 116, 49, 46, 95, 98, 108,
+        111, 99, 107, 95, 110, 97, 109, 101, 66, 55, 83, 69, 76, 69, 67, 84, 32, 97, 44, 32, 115,
+        117, 109, 40, 98, 41, 32, 70, 82, 79, 77, 32, 100, 101, 102, 97, 117, 108, 116, 46, 116,
+        49, 32, 87, 72, 69, 82, 69, 32, 97, 32, 62, 32, 51, 32, 71, 82, 79, 85, 80, 32, 66, 89, 32,
+        98, 160, 6, 67, 168, 6, 24,
+    ];
+
+    let want = || {
+        let table_id = 7;
+        let index_type = IndexType::AGGREGATING;
+        let created_on = Utc.with_ymd_and_hms(2015, 3, 9, 20, 0, 9).unwrap();
+        let original_query = "SELECT a, sum(b) FROM default.t1 WHERE a > 3 GROUP BY b".to_string();
+        let query = "SELECT a, SUM(b), t1._block_name FROM default.t1 WHERE a > 3 GROUP BY b, t1._block_name".to_string();
+
+        IndexMeta {
+            table_id,
+            index_type,
+            created_on,
+            dropped_on: None,
+            original_query,
+            query,
+            updated_on: None,
+            sync_creation: false,
+            user_defined_block_name: false,
+        }
+    };
+
+    common::test_pb_from_to(func_name!(), want())?;
+    common::test_load_old(func_name!(), index_v067.as_slice(), 67, want())?;
+
+    Ok(())
+}
diff --git a/src/meta/protos/proto/index.proto b/src/meta/protos/proto/index.proto
@@ -62,4 +62,9 @@ message IndexMeta {
   // if true, index will create after data written to databend,
   // no need execute refresh index manually.
   bool sync_creation = 7;
+
+  string original_query = 8;
+
+  /// if user already use the internal column name `_block_name` in their sql.
+  bool user_defined_block_name = 9;
 }
diff --git a/src/query/ee/Cargo.toml b/src/query/ee/Cargo.toml
@@ -14,6 +14,7 @@ test = false
 [dependencies]
 # Workspace dependencies
 async-backtrace = { workspace = true }
+databend-common-ast = { path = "../ast" }
 databend-common-base = { path = "../../common/base" }
 databend-common-catalog = { path = "../catalog" }
 databend-common-config = { path = "../config" }