Skip to content

Commit

Permalink
Refactor SQL builder to allow more granular overrides of its behavior
Browse files Browse the repository at this point in the history
  • Loading branch information
@YuriyIvon
YuriyIvon committed Jul 16, 2023
1 parent f5a32cf commit ac3e746
Show file tree
Hide file tree
Showing 14 changed files with 248 additions and 147 deletions.
127 changes: 76 additions & 51 deletions src/DatabaseBenchmark/Databases/PostgreSql/PostgreSqlJsonbQueryBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,52 +75,48 @@ protected override string BuildGroupCondition(QueryGroupCondition condition)
}
}

protected override string BuildPrimitiveCondition(QueryPrimitiveCondition condition)
protected override string BuildInCondition(QueryPrimitiveCondition condition)
{
var column = Table.Columns.FirstOrDefault(c => c.Name == condition.ColumnName);
var column = GetColumn(condition.ColumnName);

if (_queryOptions.UseGinOperators && column.Queryable)
{
if (condition.Operator == QueryPrimitiveOperator.In)
{
var rawCollection = condition.RandomizeValue
? RandomValueProvider.GetRandomValueCollection(Table.Name, condition.ColumnName, condition.ValueRandomizationRule)
: (IEnumerable<object>)condition.Value;
var rawCollection = condition.RandomizeValue
? RandomValueProvider.GetRandomValueCollection(Table.Name, condition.ColumnName, condition.ValueRandomizationRule)
: (IEnumerable<object>)condition.Value;

//Rewrite IN operator as a set of OR expressions
var orCondition = new QueryGroupCondition
{
Operator = QueryGroupOperator.Or,
Conditions = rawCollection.Select(v =>
new QueryPrimitiveCondition
{
ColumnName = condition.ColumnName,
Operator = QueryPrimitiveOperator.Equals,
Value = v
})
.ToArray()
};

return BuildGroupCondition(orCondition);
}
else if (condition.Operator == QueryPrimitiveOperator.Equals)
//Rewrite IN operator as a set of OR expressions
var orCondition = new QueryGroupCondition
{
var predicateExpression = new StringBuilder(PostgreSqlJsonbConstants.JsonbColumnName);
predicateExpression.Append(" @>");

var rawValue = condition.RandomizeValue
? RandomValueProvider.GetRandomValue(Table.Name, condition.ColumnName, condition.ValueRandomizationRule)
: condition.Value;
Operator = QueryGroupOperator.Or,
Conditions = rawCollection.Select(v =>
new QueryPrimitiveCondition
{
ColumnName = condition.ColumnName,
Operator = QueryPrimitiveOperator.Equals,
Value = v
})
.ToArray()
};

var value = rawValue == null
? "null"
: rawValue is string s
? $"\"{EscapeString(s)}\""
: rawValue.ToString();
return BuildGroupCondition(orCondition);
}
else
{
return base.BuildInCondition(condition);
}
}

predicateExpression.Append($" '{{\"{column.Name}\": {value}}}'::jsonb");
protected override string BuildComparisonCondition(QueryPrimitiveCondition condition, object value)
{
var column = GetColumn(condition.ColumnName);

return predicateExpression.ToString();
if (_queryOptions.UseGinOperators && column.Queryable)
{
if (condition.Operator == QueryPrimitiveOperator.Equals)
{
var formattedValue = FormatValue(value);
return $"{PostgreSqlJsonbConstants.JsonbColumnName} @> '{{\"{column.Name}\": {formattedValue}}}'::jsonb";
}
else
{
Expand All @@ -139,25 +135,51 @@ protected override string BuildPrimitiveCondition(QueryPrimitiveCondition condit
_ => throw new InputArgumentException($"Unknown primitive operator \"{condition.Operator}\"")
});
predicateExpression.Append(' ');

var rawValue = condition.RandomizeValue
? RandomValueProvider.GetRandomValue(Table.Name, condition.ColumnName, condition.ValueRandomizationRule)
: condition.Value;

var value = rawValue == null
? "null"
: rawValue is string s
? $"\"{EscapeString(s)}\""
: rawValue.ToString();

predicateExpression.Append(value);
predicateExpression.Append(FormatValue(value));
predicateExpression.Append('\'');

return predicateExpression.ToString();
}
}
else
{
return base.BuildComparisonCondition(condition, value);
}
}

return base.BuildPrimitiveCondition(condition);
protected override string BuildStringCondition(QueryPrimitiveCondition condition, object value)
{
var column = GetColumn(condition.ColumnName);

if (_queryOptions.UseGinOperators && column.Queryable)
{
throw new InputArgumentException("PostgreSQL jsonb operators don't support string functions");
}
else
{
return base.BuildStringCondition(condition, value);
}
}

protected override string BuildNullCondition(QueryPrimitiveCondition condition)
{
var column = GetColumn(condition.ColumnName);

if (_queryOptions.UseGinOperators && column.Queryable)
{
var basicExpression = $"{PostgreSqlJsonbConstants.JsonbColumnName} @> '{{\"{column.Name}\": null}}'::jsonb";

return condition.Operator switch
{
QueryPrimitiveOperator.Equals => basicExpression,
QueryPrimitiveOperator.NotEquals => BuildNotCondition(new string[] { basicExpression }),
_ => throw new InputArgumentException($"Primitive operator \"{condition.Operator}\" can't be used with NULL operand")
};
}
else
{
return base.BuildNullCondition(condition);
}
}

private static string BuildUnaryCondition(string @operator, string[] inputConditions)
Expand All @@ -170,6 +192,9 @@ private static string BuildUnaryCondition(string @operator, string[] inputCondit
return $"{@operator}({inputConditions.First()})";
}

private static string EscapeString(string s) => s.Replace("'", "''");
private static string FormatValue(object value) =>
value is string s
? $"\"{s.Replace("'", "''")}\""
: value.ToString();
}
}
118 changes: 72 additions & 46 deletions src/DatabaseBenchmark/Databases/Sql/SqlQueryBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,63 +199,26 @@ protected virtual string BuildNotCondition(string[] inputConditions)

protected virtual string BuildPrimitiveCondition(QueryPrimitiveCondition condition)
{
var column = Table.Columns.FirstOrDefault(c => c.Name == condition.ColumnName);

if (condition.Operator == QueryPrimitiveOperator.In)
{
var conditionExpression = new StringBuilder(BuildRegularColumnReference(condition.ColumnName));
conditionExpression.Append(' ');

var rawCollection = condition.RandomizeValue
? RandomValueProvider.GetRandomValueCollection(Table.Name, condition.ColumnName, condition.ValueRandomizationRule)
: (IEnumerable<object>)condition.Value;

if (rawCollection == null)
{
throw new InputArgumentException($"Primitive operator \"{condition.Operator}\" can't be used with NULL operand");
}

conditionExpression.Append("IN (");
conditionExpression.Append(string.Join(", ", rawCollection.Select(v => ParametersBuilder.Append(v, column.Type))));
conditionExpression.Append(')');

return conditionExpression.ToString();
return BuildInCondition(condition);
}
else
{
var conditionExpression = new StringBuilder(BuildRegularColumnReference(condition.ColumnName));
conditionExpression.Append(' ');

var rawValue = condition.RandomizeValue
var value = condition.RandomizeValue
? RandomValueProvider.GetRandomValue(Table.Name, condition.ColumnName, condition.ValueRandomizationRule)
: condition.Value;

if (rawValue != null)
if (value != null)
{
conditionExpression.Append(condition.Operator switch
if (condition.Operator == QueryPrimitiveOperator.Contains || condition.Operator == QueryPrimitiveOperator.StartsWith)
{
QueryPrimitiveOperator.Equals => "=",
QueryPrimitiveOperator.NotEquals => "<>",
QueryPrimitiveOperator.Greater => ">",
QueryPrimitiveOperator.GreaterEquals => ">=",
QueryPrimitiveOperator.Lower => "<",
QueryPrimitiveOperator.LowerEquals => "<=",
QueryPrimitiveOperator.Contains => "LIKE",
QueryPrimitiveOperator.StartsWith => "LIKE",
_ => throw new InputArgumentException($"Unknown primitive operator \"{condition.Operator}\"")
});

var value = condition.Operator switch
return BuildStringCondition(condition, value);
}
else
{
QueryPrimitiveOperator.Contains => $"%{rawValue}%",
QueryPrimitiveOperator.StartsWith => $"{rawValue}%",
_ => rawValue
};

conditionExpression.Append(' ');
conditionExpression.Append(ParametersBuilder.Append(value, column.Type));

return conditionExpression.ToString();
return BuildComparisonCondition(condition, value);
}
}
else
{
Expand All @@ -264,6 +227,69 @@ protected virtual string BuildPrimitiveCondition(QueryPrimitiveCondition conditi
}
}

protected virtual string BuildInCondition(QueryPrimitiveCondition condition)
{
var column = GetColumn(condition.ColumnName);

var conditionExpression = new StringBuilder(BuildRegularColumnReference(condition.ColumnName));
conditionExpression.Append(' ');

var rawCollection = condition.RandomizeValue
? RandomValueProvider.GetRandomValueCollection(Table.Name, condition.ColumnName, condition.ValueRandomizationRule)
: (IEnumerable<object>)condition.Value;

if (rawCollection == null)
{
throw new InputArgumentException($"Primitive operator \"{condition.Operator}\" can't be used with NULL operand");
}

conditionExpression.Append("IN (");
conditionExpression.Append(string.Join(", ", rawCollection.Select(v => ParametersBuilder.Append(v, column.Type))));
conditionExpression.Append(')');

return conditionExpression.ToString();
}

protected virtual string BuildStringCondition(QueryPrimitiveCondition condition, object value)
{
var column = GetColumn(condition.ColumnName);
var columnReference = BuildRegularColumnReference(condition.ColumnName);

var pattern = condition.Operator switch
{
QueryPrimitiveOperator.Contains => $"%{value}%",
QueryPrimitiveOperator.StartsWith => $"{value}%",
_ => throw new InputArgumentException($"Unknown string operator \"{condition.Operator}\"")
};

return $"{columnReference} LIKE {ParametersBuilder.Append(pattern, column.Type)}";
}

protected virtual string BuildComparisonCondition(QueryPrimitiveCondition condition, object value)
{
var column = GetColumn(condition.ColumnName);
var columnReference = BuildRegularColumnReference(condition.ColumnName);

var conditionExpression = new StringBuilder(columnReference);
conditionExpression.Append(' ');

conditionExpression.Append(condition.Operator switch
{
QueryPrimitiveOperator.Equals => "=",
QueryPrimitiveOperator.NotEquals => "<>",
QueryPrimitiveOperator.Greater => ">",
QueryPrimitiveOperator.GreaterEquals => ">=",
QueryPrimitiveOperator.Lower => "<",
QueryPrimitiveOperator.LowerEquals => "<=",
_ => throw new InputArgumentException($"Unknown comparison operator \"{condition.Operator}\"")
});

conditionExpression.Append(' ');
conditionExpression.Append(ParametersBuilder.Append(value, column.Type));

return conditionExpression.ToString();
}

protected virtual string BuildLimit()
{
var expression = new StringBuilder();
Expand Down
1 change: 0 additions & 1 deletion tests/DatabaseBenchmark.Tests/Common/CachedDistinctValuesProviderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
using DatabaseBenchmark.Core.Interfaces;
using NSubstitute;
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using Xunit;

Expand Down
7 changes: 4 additions & 3 deletions tests/DatabaseBenchmark.Tests/Databases/ClickHouseQueryBuilderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
using DatabaseBenchmark.Databases.Sql;
using DatabaseBenchmark.Model;
using DatabaseBenchmark.Tests.Utils;
using System.Linq;
using Xunit;

namespace DatabaseBenchmark.Tests.Databases
Expand Down Expand Up @@ -32,15 +31,17 @@ public void BuildQueryAllArguments()

var normalizedQueryText = queryText.NormalizeSpaces();
Assert.Equal("SELECT Category, SubCategory, SUM(Price) TotalPrice FROM Sample"
+ " WHERE (Category = @p0 AND SubCategory IS NULL AND Rating >= @p1)"
+ " WHERE (Category = @p0 AND SubCategory IS NULL AND Rating >= @p1 AND (Name LIKE @p2 OR Name LIKE @p3))"
+ " GROUP BY Category, SubCategory"
+ " ORDER BY Category ASC, SubCategory ASC"
+ " LIMIT 100 OFFSET 10", normalizedQueryText);

var reference = new SqlQueryParameter[]
{
new ('@', "p0", "ABC", ColumnType.String),
new ('@', "p1", 5.0, ColumnType.Double)
new ('@', "p1", 5.0, ColumnType.Double),
new ('@', "p2", "A%", ColumnType.String),
new ('@', "p3", "%B%", ColumnType.String)
};

Assert.Equal(reference, parametersBuilder.Parameters);
Expand Down
10 changes: 6 additions & 4 deletions tests/DatabaseBenchmark.Tests/Databases/CosmosDbQueryBuilderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,19 @@ public void BuildQueryAllArguments()
//Please note than in fact in CosmosDB GROUP BY and ORDER BY are mutually exclusive
var normalizedQueryText = queryText.NormalizeSpaces();
Assert.Equal("SELECT Sample.Category, Sample.SubCategory, SUM(Sample.Price) TotalPrice FROM Sample"
+ " WHERE (Sample.Category = @p0 AND IS_NULL(Sample.SubCategory) AND Sample.Rating >= @p1)"
+ " WHERE (Sample.Category = @p0 AND IS_NULL(Sample.SubCategory) AND Sample.Rating >= @p1 AND (Sample.Name LIKE @p2 OR Sample.Name LIKE @p3))"
+ " GROUP BY Sample.Category, Sample.SubCategory"
+ " ORDER BY Sample.Category ASC, Sample.SubCategory ASC"
+ " OFFSET @p2 LIMIT @p3", normalizedQueryText);
+ " OFFSET @p4 LIMIT @p5", normalizedQueryText);

var reference = new SqlQueryParameter[]
{
new ('@', "p0", "ABC", ColumnType.String),
new ('@', "p1", 5.0, ColumnType.Double),
new ('@', "p2", 10, ColumnType.Integer),
new ('@', "p3", 100, ColumnType.Integer)
new ('@', "p2", "A%", ColumnType.String),
new ('@', "p3", "%B%", ColumnType.String),
new ('@', "p4", 10, ColumnType.Integer),
new ('@', "p5", 100, ColumnType.Integer)
};

Assert.Equal(reference, parametersBuilder.Parameters);
Expand Down
4 changes: 2 additions & 2 deletions tests/DatabaseBenchmark.Tests/Databases/ElasticsearchQueryBuilderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public void BuildQueryAllArguments()
Assert.Equal("{\"aggs\":{\"grouping\":{\"aggs\":{\"TotalPrice\":{\"sum\":{\"field\":\"Price\"}}},\"composite\":{\"size\":10000,\"sources\":[{\"Category\":{\"terms\":{\"field\":\"Category\",\"order\":\"asc\"}}},{\"SubCategory\":{\"terms\":{\"field\":\"SubCategory\",\"order\":\"asc\"}}}]}}}," +
"\"fields\":[\"Category\",\"SubCategory\"]," +
"\"from\":10," +
"\"query\":{\"bool\":{\"must\":[{\"term\":{\"Category\":{\"value\":\"ABC\"}}},{\"bool\":{\"must_not\":[{\"exists\":{\"field\":\"SubCategory\"}}]}},{\"range\":{\"Rating\":{\"gte\":5.0}}}]}}," +
"\"query\":{\"bool\":{\"must\":[{\"term\":{\"Category\":{\"value\":\"ABC\"}}},{\"bool\":{\"must_not\":[{\"exists\":{\"field\":\"SubCategory\"}}]}},{\"range\":{\"Rating\":{\"gte\":5.0}}},{\"bool\":{\"should\":[{\"wildcard\":{\"Name\":{\"value\":\"A*\"}}},{\"wildcard\":{\"Name\":{\"value\":\"*B*\"}}}]}}]}}," +
"\"size\":100}",
rawQuery);
}
Expand Down Expand Up @@ -72,7 +72,7 @@ public void BuildQueryAllArgumentsIncludePartial()
Assert.Equal("{\"aggs\":{\"grouping\":{\"aggs\":{\"TotalPrice\":{\"sum\":{\"field\":\"Price\"}}},\"composite\":{\"size\":10000,\"sources\":[{\"Category\":{\"terms\":{\"field\":\"Category\",\"order\":\"asc\"}}},{\"SubCategory\":{\"terms\":{\"field\":\"SubCategory\",\"order\":\"asc\"}}}]}}}," +
"\"fields\":[\"Category\",\"SubCategory\"]," +
"\"from\":10," +
"\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must_not\":[{\"exists\":{\"field\":\"SubCategory\"}}]}},{\"range\":{\"Rating\":{\"gte\":5.0}}}]}}," +
"\"query\":{\"bool\":{\"must\":[{\"bool\":{\"must_not\":[{\"exists\":{\"field\":\"SubCategory\"}}]}},{\"range\":{\"Rating\":{\"gte\":5.0}}},{\"bool\":{\"should\":[{\"wildcard\":{\"Name\":{\"value\":\"A*\"}}},{\"wildcard\":{\"Name\":{\"value\":\"*B*\"}}}]}}]}}," +
"\"size\":100}",
rawQuery);
}
Expand Down
Loading

0 comments on commit ac3e746

Please sign in to comment.