Skip to content

Commit

Permalink
Merge branch 'main' into split2
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhang2014
zhang2014 authored Nov 29, 2024
2 parents 58275f3 + 8cbda70 commit c8e9076
Show file tree
Hide file tree
Showing 61 changed files with 1,439 additions and 702 deletions.
4 changes: 2 additions & 2 deletions .github/actions/setup_bendsql/action.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ runs:
if bendsql --version; then
exit 0
fi
curl --retry 5 -Lo /tmp/bendsql.tar.gz https://github.com/databendlabs/bendsql/releases/download/v0.18.3/bendsql-x86_64-unknown-linux-gnu.tar.gz
curl --retry 5 -Lo /tmp/bendsql.tar.gz https://github.com/databendlabs/bendsql/releases/download/v0.23.2/bendsql-x86_64-unknown-linux-gnu.tar.gz
tar -xzf /tmp/bendsql.tar.gz -C /tmp
mv /tmp/bendsql /usr/local/bin/bendsql
bendsql --version
Expand All @@ -21,7 +21,7 @@ runs:
if bendsql --version; then
exit 0
fi
curl --retry 5 -Lo /tmp/bendsql.tar.gz https://github.com/databendlabs/bendsql/releases/download/v0.18.3/bendsql-x86_64-apple-darwin.tar.gz
curl --retry 5 -Lo /tmp/bendsql.tar.gz https://github.com/databendlabs/bendsql/releases/download/v0.23.2/bendsql-x86_64-apple-darwin.tar.gz
tar -xzf /tmp/bendsql.tar.gz -C /tmp
mv /tmp/bendsql /usr/local/bin/bendsql
bendsql --version
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

187 changes: 174 additions & 13 deletions src/meta/api/src/schema_api_test_suite.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6416,7 +6416,26 @@ impl SchemaApiTestSuite {
let req = CreateVirtualColumnReq {
create_option: CreateOption::Create,
name_ident: name_ident.clone(),
virtual_columns: vec!["variant:k1".to_string(), "variant[1]".to_string()],
virtual_columns: vec![
(
"variant:k1".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant)),
),
(
"variant[1]".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant)),
),
(
"variant:k1:k2".to_string(),
TableDataType::Nullable(Box::new(TableDataType::String)),
),
(
"variant:k1:k3".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Number(
NumberDataType::UInt64,
))),
),
],
};

mt.create_virtual_column(req.clone()).await?;
Expand All @@ -6425,7 +6444,26 @@ impl SchemaApiTestSuite {
let req = CreateVirtualColumnReq {
create_option: CreateOption::Create,
name_ident: name_ident.clone(),
virtual_columns: vec!["variant:k1".to_string(), "variant[1]".to_string()],
virtual_columns: vec![
(
"variant:k1".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant)),
),
(
"variant[1]".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant)),
),
(
"variant:k1:k2".to_string(),
TableDataType::Nullable(Box::new(TableDataType::String)),
),
(
"variant:k1:k3".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Number(
NumberDataType::UInt64,
))),
),
],
};

let res = mt.create_virtual_column(req).await;
Expand All @@ -6439,8 +6477,24 @@ impl SchemaApiTestSuite {
let res = mt.list_virtual_columns(req).await?;
assert_eq!(1, res.len());
assert_eq!(res[0].virtual_columns, vec![
"variant:k1".to_string(),
"variant[1]".to_string(),
(
"variant:k1".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant))
),
(
"variant[1]".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant))
),
(
"variant:k1:k2".to_string(),
TableDataType::Nullable(Box::new(TableDataType::String)),
),
(
"variant:k1:k3".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Number(
NumberDataType::UInt64
))),
),
]);

let req = ListVirtualColumnsReq::new(&tenant, Some(u64::MAX));
Expand All @@ -6454,7 +6508,26 @@ impl SchemaApiTestSuite {
let req = UpdateVirtualColumnReq {
if_exists: false,
name_ident: name_ident.clone(),
virtual_columns: vec!["variant:k2".to_string(), "variant[2]".to_string()],
virtual_columns: vec![
(
"variant:k2".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant)),
),
(
"variant[2]".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant)),
),
(
"variant:k2:k3".to_string(),
TableDataType::Nullable(Box::new(TableDataType::String)),
),
(
"variant:k2:k4".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Number(
NumberDataType::UInt64,
))),
),
],
};

mt.update_virtual_column(req).await?;
Expand All @@ -6467,8 +6540,24 @@ impl SchemaApiTestSuite {
let res = mt.list_virtual_columns(req).await?;
assert_eq!(1, res.len());
assert_eq!(res[0].virtual_columns, vec![
"variant:k2".to_string(),
"variant[2]".to_string(),
(
"variant:k2".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant))
),
(
"variant[2]".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant))
),
(
"variant:k2:k3".to_string(),
TableDataType::Nullable(Box::new(TableDataType::String)),
),
(
"variant:k2:k4".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Number(
NumberDataType::UInt64
))),
),
]);
}

Expand All @@ -6495,7 +6584,26 @@ impl SchemaApiTestSuite {
let req = UpdateVirtualColumnReq {
if_exists: false,
name_ident: name_ident.clone(),
virtual_columns: vec!["variant:k3".to_string(), "variant[3]".to_string()],
virtual_columns: vec![
(
"variant:k3".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant)),
),
(
"variant[3]".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant)),
),
(
"variant:k3:k4".to_string(),
TableDataType::Nullable(Box::new(TableDataType::String)),
),
(
"variant:k3:k5".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Number(
NumberDataType::UInt64,
))),
),
],
};

let res = mt.update_virtual_column(req).await;
Expand All @@ -6507,7 +6615,26 @@ impl SchemaApiTestSuite {
let req = CreateVirtualColumnReq {
create_option: CreateOption::Create,
name_ident: name_ident.clone(),
virtual_columns: vec!["variant:k1".to_string(), "variant[1]".to_string()],
virtual_columns: vec![
(
"variant:k1".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant)),
),
(
"variant[1]".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant)),
),
(
"variant:k1:k4".to_string(),
TableDataType::Nullable(Box::new(TableDataType::String)),
),
(
"variant:k1:k5".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Number(
NumberDataType::UInt64,
))),
),
],
};

mt.create_virtual_column(req.clone()).await?;
Expand All @@ -6517,14 +6644,39 @@ impl SchemaApiTestSuite {
let res = mt.list_virtual_columns(req).await?;
assert_eq!(1, res.len());
assert_eq!(res[0].virtual_columns, vec![
"variant:k1".to_string(),
"variant[1]".to_string(),
(
"variant:k1".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant))
),
(
"variant[1]".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant))
),
(
"variant:k1:k4".to_string(),
TableDataType::Nullable(Box::new(TableDataType::String)),
),
(
"variant:k1:k5".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Number(
NumberDataType::UInt64
))),
),
]);

let req = CreateVirtualColumnReq {
create_option: CreateOption::CreateOrReplace,
name_ident: name_ident.clone(),
virtual_columns: vec!["variant:k2".to_string()],
virtual_columns: vec![
(
"variant:k2".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant)),
),
(
"variant:k3".to_string(),
TableDataType::Nullable(Box::new(TableDataType::String)),
),
],
};

mt.create_virtual_column(req.clone()).await?;
Expand All @@ -6533,7 +6685,16 @@ impl SchemaApiTestSuite {

let res = mt.list_virtual_columns(req).await?;
assert_eq!(1, res.len());
assert_eq!(res[0].virtual_columns, vec!["variant:k2".to_string(),]);
assert_eq!(res[0].virtual_columns, vec![
(
"variant:k2".to_string(),
TableDataType::Nullable(Box::new(TableDataType::Variant))
),
(
"variant:k3".to_string(),
TableDataType::Nullable(Box::new(TableDataType::String)),
)
]);
}

Ok(())
Expand Down
7 changes: 4 additions & 3 deletions src/meta/app/src/schema/virtual_column.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ use std::fmt::Formatter;

use chrono::DateTime;
use chrono::Utc;
use databend_common_expression::TableDataType;
use databend_common_meta_types::MetaId;

use super::CreateOption;
Expand All @@ -29,7 +30,7 @@ use crate::tenant::ToTenant;
pub struct VirtualColumnMeta {
pub table_id: MetaId,

pub virtual_columns: Vec<String>,
pub virtual_columns: Vec<(String, TableDataType)>,
pub created_on: DateTime<Utc>,
pub updated_on: Option<DateTime<Utc>>,
}
Expand All @@ -38,7 +39,7 @@ pub struct VirtualColumnMeta {
pub struct CreateVirtualColumnReq {
pub create_option: CreateOption,
pub name_ident: VirtualColumnIdent,
pub virtual_columns: Vec<String>,
pub virtual_columns: Vec<(String, TableDataType)>,
}

impl Display for CreateVirtualColumnReq {
Expand All @@ -56,7 +57,7 @@ impl Display for CreateVirtualColumnReq {
pub struct UpdateVirtualColumnReq {
pub if_exists: bool,
pub name_ident: VirtualColumnIdent,
pub virtual_columns: Vec<String>,
pub virtual_columns: Vec<(String, TableDataType)>,
}

impl Display for UpdateVirtualColumnReq {
Expand Down
4 changes: 2 additions & 2 deletions src/meta/binaries/metactl/import.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ async fn import_v003(
raft_config: RaftConfig,
lines: impl IntoIterator<Item = Result<String, io::Error>>,
) -> anyhow::Result<Option<LogId>> {
let db = init_get_sled_db(raft_config.raft_dir.clone(), raft_config.sled_cache_size());
let db = init_get_sled_db(raft_config.raft_dir.clone(), 1024 * 1024 * 1024);

let mut n = 0;
let mut max_log_id: Option<LogId> = None;
Expand Down Expand Up @@ -386,7 +386,7 @@ async fn init_new_cluster(
fn clear(args: &ImportArgs) -> anyhow::Result<()> {
eprintln!();
eprintln!("Clear All Sled Trees Before Import:");
let db = init_get_sled_db(args.raft_dir.clone().unwrap(), 64 * 1024 * 1024 * 1024);
let db = init_get_sled_db(args.raft_dir.clone().unwrap(), 1024 * 1024 * 1024);

let tree_names = db.tree_names();
for n in tree_names.iter() {
Expand Down
2 changes: 1 addition & 1 deletion src/meta/process/src/process_meta_dir.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ pub fn process_sled_db<F>(config: &Config, convert: F) -> anyhow::Result<()>
where F: Fn(RaftStoreEntry) -> Result<Option<RaftStoreEntry>, anyhow::Error> {
let raft_config = &config.raft_config;

let db = init_get_sled_db(raft_config.raft_dir.clone(), 64 * 1024 * 1024 * 1024);
let db = init_get_sled_db(raft_config.raft_dir.clone(), 1024 * 1024 * 1024);

let mut tree_names = db.tree_names();
tree_names.sort();
Expand Down
1 change: 1 addition & 0 deletions src/meta/proto-conv/src/util.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
(109, "2024-08-29: Refactor: ProcedureMeta add arg_names"),
(110, "2024-09-18: Add: database.proto: DatabaseMeta.gc_in_progress"),
(111, "2024-11-13: Add: Enable AWS Glue as an Apache Iceberg type when creating a catalog."),
(112, "2024-11-28: Add: virtual_column add data_types field"),
// Dear developer:
// If you're gonna add a new metadata version, you'll have to add a test for it.
// You could just copy an existing test file(e.g., `../tests/it/v024_table_meta.rs`)
Expand Down
Loading

0 comments on commit c8e9076

Please sign in to comment.