Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add introspectSql command #4941

Merged
merged 36 commits into from
Aug 5, 2024
Merged
Show file tree
Hide file tree
Changes from 29 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
35d8018
wip: vertical slice of introspect sql
Weakky Jul 4, 2024
004522f
fix rebase
Weakky Jul 24, 2024
61b04c8
add sqlite support (unnamed & unknom type for params)
Weakky Jul 24, 2024
68e7669
add tests
Weakky Jul 24, 2024
07f1d43
enhance tests
Weakky Jul 24, 2024
c06be19
rename internals
Weakky Jul 24, 2024
ac7c981
add sqlite support in quaint as well
Weakky Jul 24, 2024
aed1fd3
resolve enum names for pg
Weakky Jul 25, 2024
d27bf46
add enum test
Weakky Jul 25, 2024
cc3e9ea
add basic sqlite tests
Weakky Jul 25, 2024
c7a13be
split tests in different files
Weakky Jul 25, 2024
0a2fc83
add rpc command
Weakky Jul 25, 2024
94c2717
add native types tests
Weakky Jul 25, 2024
8619e38
add few more tests
Weakky Jul 25, 2024
dfed2f0
fix mysql56 test
Weakky Jul 25, 2024
741b5e7
fix crdb tests
Weakky Jul 25, 2024
13f1cd2
refactor & fix tests
Weakky Jul 26, 2024
b2b600c
add source to output
Weakky Jul 26, 2024
664740d
clippy fix
Weakky Jul 26, 2024
239c912
add more mysql & sqlite tests
Weakky Jul 26, 2024
23edba6
add more sqlite tests
Weakky Jul 26, 2024
b9fe531
add support for magic comments
Weakky Jul 29, 2024
c285da0
fix fmt
Weakky Jul 29, 2024
40f60b4
fix tests
Weakky Jul 29, 2024
0fa5f05
make documentation optional
Weakky Jul 29, 2024
93ecf00
cleanup
Weakky Jul 29, 2024
6e50b79
exclude vitess (overlaps with mysql8 but returns different results)
Weakky Jul 29, 2024
9d9441c
make mssql implementation explicitly unimplemented
Weakky Jul 29, 2024
b067a86
fixes unit test
Weakky Jul 29, 2024
921f6bc
small cleanup
Weakky Jul 30, 2024
405f504
add pg/crdb insert test back
Weakky Jul 30, 2024
dc10635
review cleanup
Weakky Jul 30, 2024
59e50b6
make mysql 5.6 & 5.7 return unknown for params
Weakky Jul 30, 2024
4dcdf8b
add typedSql preview feature
Weakky Aug 2, 2024
e3dc131
make typedsql feat hidden
Weakky Aug 5, 2024
9312993
add unhandled multi-line comment test
Weakky Aug 5, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

68 changes: 54 additions & 14 deletions libs/test-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ mod diagnose_migration_history;

use anyhow::Context;
use colored::Colorize;
use psl::parse_configuration;
use schema_connector::BoxFuture;
use schema_core::json_rpc::types::*;
use std::{fmt, fs::File, io::Read, str::FromStr, sync::Arc};
Expand All @@ -24,6 +25,17 @@ enum Command {
#[structopt(long)]
composite_type_depth: Option<isize>,
},
/// Parse SQL queries and returns type information.
IntrospectSql {
/// URL of the database to introspect.
#[structopt(long)]
url: Option<String>,
/// Path to the schema file to introspect for.
#[structopt(long = "file-path")]
file_path: Option<String>,
/// The SQL query to introspect.
query_file_path: String,
},
/// Generate DMMF from a schema, or directly from a database URL.
Dmmf(DmmfCommand),
/// Push a prisma schema directly to the database.
Expand Down Expand Up @@ -184,25 +196,39 @@ async fn main() -> anyhow::Result<()> {
Command::Dmmf(cmd) => generate_dmmf(&cmd).await?,
Command::SchemaPush(cmd) => schema_push(&cmd).await?,
Command::MigrateDiff(cmd) => migrate_diff(&cmd).await?,
Command::IntrospectSql {
url,
file_path,
query_file_path,
} => {
let schema = schema_from_args(url.as_deref(), file_path.as_deref())?;
let config = parse_configuration(&schema).unwrap();
let api = schema_core::schema_api(Some(schema.clone()), None)?;
let query_str = std::fs::read_to_string(query_file_path)?;

let res = api
.introspect_sql(IntrospectSqlParams {
url: config
.first_datasource()
.load_url(|key| std::env::var(key).ok())
.unwrap(),
force: false,
queries: vec![SqlQueryInput {
name: "query".to_string(),
source: query_str,
}],
})
.await
.map_err(|err| anyhow::anyhow!("{err:?}"))?;

println!("{}", serde_json::to_string_pretty(&res).unwrap());
}
Command::Introspect {
url,
file_path,
composite_type_depth,
} => {
if url.as_ref().xor(file_path.as_ref()).is_none() {
anyhow::bail!(
"{}",
"Exactly one of --url or --file-path must be provided".bold().red()
);
}

let schema = if let Some(file_path) = &file_path {
read_datamodel_from_file(file_path)?
} else if let Some(url) = &url {
minimal_schema_from_url(url)?
} else {
unreachable!()
};
let schema = schema_from_args(url.as_deref(), file_path.as_deref())?;

let base_directory_path = file_path
.as_ref()
Expand Down Expand Up @@ -292,6 +318,20 @@ async fn main() -> anyhow::Result<()> {
Ok(())
}

fn schema_from_args(url: Option<&str>, file_path: Option<&str>) -> anyhow::Result<String> {
if let Some(url) = url {
let schema = minimal_schema_from_url(url)?;

Ok(schema)
} else if let Some(file_path) = file_path {
let schema = read_datamodel_from_file(file_path)?;

Ok(schema)
} else {
anyhow::bail!("Please provide one of --url or --file-path")
}
}

fn read_datamodel_from_file(path: &str) -> std::io::Result<String> {
use std::path::Path;

Expand Down
3 changes: 2 additions & 1 deletion psl/parser-database/src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1468,7 +1468,8 @@ impl ScalarType {
matches!(self, ScalarType::Bytes)
}

pub(crate) fn try_from_str(s: &str, ignore_case: bool) -> Option<ScalarType> {
/// Tries to parse a scalar type from a string.
pub fn try_from_str(s: &str, ignore_case: bool) -> Option<ScalarType> {
match ignore_case {
true => match s.to_lowercase().as_str() {
"int" => Some(ScalarType::Int),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -318,7 +318,7 @@ impl Connector for PostgresDatamodelConnector {
DoublePrecision => ScalarType::Float,
// Decimal
Decimal(_) => ScalarType::Decimal,
Money => ScalarType::Float,
Money => ScalarType::Decimal,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The mapping was incorrect. I spotted it as we're using this function for native type tests

// DateTime
Timestamp(_) => ScalarType::DateTime,
Timestamptz(_) => ScalarType::DateTime,
Expand Down
2 changes: 2 additions & 0 deletions quaint/src/connector.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ pub mod external;
pub mod metrics;
#[cfg(native)]
pub mod native;
mod parsed_query;
mod queryable;
mod result_set;
#[cfg(any(feature = "mssql-native", feature = "postgresql-native", feature = "mysql-native"))]
Expand All @@ -32,6 +33,7 @@ pub use connection_info::*;
pub use native::*;

pub use external::*;
pub use parsed_query::*;
pub use queryable::*;
pub use transaction::*;

Expand Down
6 changes: 5 additions & 1 deletion quaint/src/connector/mssql/native/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ mod conversion;
mod error;

pub(crate) use crate::connector::mssql::MssqlUrl;
use crate::connector::{timeout, IsolationLevel, Transaction, TransactionOptions};
use crate::connector::{timeout, IsolationLevel, ParsedRawQuery, Transaction, TransactionOptions};

use crate::{
ast::{Query, Value},
Expand Down Expand Up @@ -183,6 +183,10 @@ impl Queryable for Mssql {
self.query_raw(sql, params).await
}

async fn parse_raw_query(&self, _sql: &str) -> crate::Result<ParsedRawQuery> {
unimplemented!("SQL Server support for raw query parsing is not implemented yet.")
}

async fn execute(&self, q: Query<'_>) -> crate::Result<u64> {
let (sql, params) = visitor::Mssql::build(q)?;
self.execute_raw(&sql, &params[..]).await
Expand Down
21 changes: 20 additions & 1 deletion quaint/src/connector/mysql/native/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ mod conversion;
mod error;

pub(crate) use crate::connector::mysql::MysqlUrl;
use crate::connector::{timeout, ColumnType, IsolationLevel};
use crate::connector::{timeout, ColumnType, IsolationLevel, ParsedRawColumn, ParsedRawParameter, ParsedRawQuery};

use crate::{
ast::{Query, Value},
Expand Down Expand Up @@ -247,6 +247,25 @@ impl Queryable for Mysql {
self.query_raw(sql, params).await
}

async fn parse_raw_query(&self, sql: &str) -> crate::Result<ParsedRawQuery> {
self.prepared(sql, |stmt| async move {
let columns = stmt
.columns()
.iter()
.map(|col| ParsedRawColumn::new_named(col.name_str(), col))
.collect();
let parameters = stmt
.params()
.iter()
.enumerate()
.map(|(idx, col)| ParsedRawParameter::new_unnamed(idx, col))
.collect();

Ok(ParsedRawQuery { columns, parameters })
})
.await
}

async fn execute(&self, q: Query<'_>) -> crate::Result<u64> {
let (sql, params) = visitor::Mysql::build(q)?;
self.execute_raw(&sql, &params).await
Expand Down
73 changes: 73 additions & 0 deletions quaint/src/connector/parsed_query.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
use std::borrow::Cow;

use super::ColumnType;

#[derive(Debug)]
pub struct ParsedRawQuery {
pub parameters: Vec<ParsedRawParameter>,
pub columns: Vec<ParsedRawColumn>,
}

#[derive(Debug)]
pub struct ParsedRawParameter {
pub name: String,
pub typ: ColumnType,
pub enum_name: Option<String>,
}

#[derive(Debug)]
pub struct ParsedRawColumn {
pub name: String,
pub typ: ColumnType,
pub enum_name: Option<String>,
}

impl ParsedRawParameter {
pub fn new_named<'a>(name: impl Into<Cow<'a, str>>, typ: impl Into<ColumnType>) -> Self {
let name: Cow<'_, str> = name.into();

Self {
name: name.into_owned(),
typ: typ.into(),
enum_name: None,
}
}

pub fn new_unnamed(idx: usize, typ: impl Into<ColumnType>) -> Self {
Self {
name: format!("_{idx}"),
typ: typ.into(),
enum_name: None,
}
}

pub fn with_enum_name(mut self, enum_name: Option<String>) -> Self {
self.enum_name = enum_name;
self
}
}

impl ParsedRawColumn {
pub fn new_named<'a>(name: impl Into<Cow<'a, str>>, typ: impl Into<ColumnType>) -> Self {
let name: Cow<'_, str> = name.into();

Self {
name: name.into_owned(),
typ: typ.into(),
enum_name: None,
}
}

pub fn new_unnamed(idx: usize, typ: impl Into<ColumnType>) -> Self {
Self {
name: format!("_{idx}"),
typ: typ.into(),
enum_name: None,
}
}

pub fn with_enum_name(mut self, enum_name: Option<String>) -> Self {
self.enum_name = enum_name;
self
}
}
6 changes: 6 additions & 0 deletions quaint/src/connector/postgres/native/column_type.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,12 @@ macro_rules! create_pg_mapping {
}
}
}

impl From<&PostgresType> for ColumnType {
fn from(ty: &PostgresType) -> ColumnType {
PGColumnType::from_pg_type(&ty).into()
}
}
};
}

Expand Down
45 changes: 44 additions & 1 deletion quaint/src/connector/postgres/native/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ mod error;

pub(crate) use crate::connector::postgres::url::PostgresUrl;
use crate::connector::postgres::url::{Hidden, SslAcceptMode, SslParams};
use crate::connector::{timeout, ColumnType, IsolationLevel, Transaction};
use crate::connector::{
timeout, ColumnType, IsolationLevel, ParsedRawColumn, ParsedRawParameter, ParsedRawQuery, Transaction,
};
use crate::error::NativeErrorKind;

use crate::{
Expand All @@ -22,6 +24,7 @@ use futures::{future::FutureExt, lock::Mutex};
use lru_cache::LruCache;
use native_tls::{Certificate, Identity, TlsConnector};
use postgres_native_tls::MakeTlsConnector;
use postgres_types::{Kind as PostgresKind, Type as PostgresType};
use std::hash::{DefaultHasher, Hash, Hasher};
use std::{
borrow::Borrow,
Expand Down Expand Up @@ -471,6 +474,46 @@ impl Queryable for PostgreSql {
.await
}

async fn parse_raw_query(&self, sql: &str) -> crate::Result<ParsedRawQuery> {
let stmt = self.fetch_cached(sql, &[]).await?;
let mut columns: Vec<ParsedRawColumn> = Vec::with_capacity(stmt.columns().len());
let mut parameters: Vec<ParsedRawParameter> = Vec::with_capacity(stmt.params().len());

async fn infer_type(this: &PostgreSql, ty: &PostgresType) -> crate::Result<(ColumnType, Option<String>)> {
let column_type = ColumnType::from(ty);

match ty.kind() {
PostgresKind::Enum => {
let enum_name = this
.query_raw("SELECT typname FROM pg_type WHERE oid = $1;", &[Value::int64(ty.oid())])
.await?
.into_single()?
.at(0)
.expect("could not find enum name")
.to_string()
.expect("enum name is not a string");

Ok((column_type, Some(enum_name)))
}
Weakky marked this conversation as resolved.
Show resolved Hide resolved
_ => Ok((column_type, None)),
}
}

for col in stmt.columns() {
let (typ, enum_name) = infer_type(self, col.type_()).await?;

columns.push(ParsedRawColumn::new_named(col.name(), typ).with_enum_name(enum_name));
}

for param in stmt.params() {
let (typ, enum_name) = infer_type(self, param).await?;

parameters.push(ParsedRawParameter::new_named(param.name(), typ).with_enum_name(enum_name));
}

Ok(ParsedRawQuery { columns, parameters })
}

async fn execute(&self, q: Query<'_>) -> crate::Result<u64> {
let (sql, params) = visitor::Postgres::build(q)?;

Expand Down
5 changes: 4 additions & 1 deletion quaint/src/connector/queryable.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use super::{IsolationLevel, ResultSet, Transaction};
use super::{IsolationLevel, ParsedRawQuery, ResultSet, Transaction};
use crate::ast::*;
use async_trait::async_trait;

Expand Down Expand Up @@ -57,6 +57,9 @@ pub trait Queryable: Send + Sync {
/// parsing or normalization.
async fn version(&self) -> crate::Result<Option<String>>;

/// Prepares a statement and returns type information.
async fn parse_raw_query(&self, sql: &str) -> crate::Result<ParsedRawQuery>;

/// Returns false, if connection is considered to not be in a working state.
fn is_healthy(&self) -> bool;

Expand Down
Loading
Loading