Skip to content

Migrations: Optimise sortable value population for date properties #22547

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Zeegaan merged 2 commits into from
Apr 22, 2026
+97 −53
Merged

Migrations: Optimise sortable value population for date properties #22547

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2e9700c
Optimise the populate sortable column migration.
AndyButland Apr 21, 2026
e069eb2
Further optimisation from code review feedback.
AndyButland Apr 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 97 additions & 53 deletions ...co.Infrastructure/Migrations/Upgrade/V_17_3_0/PopulateSortableValueForDatePropertyData.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@ namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_17_3_0;
/// </summary>
public class PopulateSortableValueForDatePropertyData : AsyncMigrationBase
{
// Updates are performed in batches to keep individual command durations bounded and to avoid
// holding locks across the entire table for the full duration of the migration.
private const int BatchSize = 10000;

// Property editor aliases that store date/time as JSON and implement IDataValueSortable.
private static readonly string[] _dateTimePropertyEditorAliases =
[
Expand All @@ -37,6 +41,7 @@ public PopulateSortableValueForDatePropertyData(
/// <inheritdoc/>
protected override Task MigrateAsync()
{
EnsureLongCommandTimeout(Database);
ExecuteMigration(Database, DatabaseType, _logger);
return Task.CompletedTask;
}
Expand All @@ -50,12 +55,43 @@ protected override Task MigrateAsync()
/// <returns>The number of rows affected.</returns>
public static int ExecuteMigration(IUmbracoDatabase database, DatabaseType databaseType, ILogger logger)
{
// Build the IN clause for the property editor aliases.
var aliasesInClause = string.Join(", ", _dateTimePropertyEditorAliases.Select(a => $"'{a}'"));
// Resolve the relevant property type ids up front so the UPDATE can filter with a static IN list.
// A subquery here causes SQL Server to pick a scan-with-row-goal plan for the batched UPDATE,
// evaluating TRY_CAST(JSON_VALUE(textValue, '$.date')) on every row of umbracoPropertyData before
// the semi-join prunes anything — poor performance on large tables even when no rows ultimately match.
// A static IN list lets the optimizer drive the query off the IX_umbracoPropertyData_PropertyTypeId
// index, so TRY_CAST / datetime() only evaluate on rows already narrowed down by property type.
List<int> propertyTypeIds = GetSortablePropertyTypeIds(database);
if (propertyTypeIds.Count == 0)
{
logger.LogInformation(
"Skipping sortableValue population; no property types using date/time editors were found.");
return 0;
}

logger.LogInformation(
"Populating sortableValue for property data across {PropertyTypeCount} date/time property type(s).",
propertyTypeIds.Count);

var idsInClause = string.Join(", ", propertyTypeIds);

return databaseType == DatabaseType.SQLite
? MigrateSQLite(database, aliasesInClause, logger)
: MigrateSqlServer(database, aliasesInClause, logger);
? MigrateSQLite(database, idsInClause, logger)
: MigrateSqlServer(database, idsInClause, logger);
}

private static List<int> GetSortablePropertyTypeIds(IUmbracoDatabase database)
{
var aliasesInClause = string.Join(", ", _dateTimePropertyEditorAliases.Select(a => $"'{a}'"));
var sql = $@"
SELECT id
FROM cmsPropertyType
WHERE dataTypeId IN (
SELECT nodeId
FROM umbracoDataType
WHERE propertyEditorAlias IN ({aliasesInClause})
)";
return database.Fetch<int>(sql);
}

/// <summary>
Expand All @@ -64,36 +100,26 @@ public static int ExecuteMigration(IUmbracoDatabase database, DatabaseType datab
/// <remarks>
/// The JSON format is: {"date":"2025-11-05T15:31:00+00:00","timeZone":"UTC"}
/// We extract the date, parse it as datetimeoffset, convert to UTC, and format as ISO 8601.
/// The resulting format is: 2025-11-05T15:31:00.0000000+00:00
/// The resulting format is: 2025-11-05T15:31:00.0000000+00:00.
/// TRY_CAST is used so rows with unparseable dates are skipped rather than aborting the migration;
/// the same guard in the WHERE clause ensures the batch loop always makes progress.
/// The parsed datetimeoffset is computed once via CROSS APPLY and reused for both the filter
/// and the assignment, avoiding a second JSON_VALUE + TRY_CAST per row.
/// </remarks>
private static int MigrateSqlServer(IUmbracoDatabase database, string aliasesInClause, ILogger logger)
private static int MigrateSqlServer(IUmbracoDatabase database, string propertyTypeIdsInClause, ILogger logger)
{
// SQL Server: Use ISJSON to validate JSON before parsing, then use JSON_VALUE to extract the date,
// CAST to datetimeoffset, SWITCHOFFSET to convert to UTC, and CONVERT with style 127 for ISO 8601 format.
// Style 127 produces: yyyy-MM-ddTHH:mm:ss.nnnnnnn or yyyy-MM-ddTHH:mm:ss.nnnnnnn+00:00
var sql = $@"
UPDATE umbracoPropertyData
SET sortableValue = CONVERT(varchar(50), SWITCHOFFSET(CAST(JSON_VALUE(textValue, '$.date') AS datetimeoffset), '+00:00'), 127)
WHERE propertyTypeId IN (
SELECT id
FROM cmsPropertyType
WHERE dataTypeId IN (
SELECT nodeId
FROM umbracoDataType
WHERE propertyEditorAlias IN ({aliasesInClause})
)
)
AND textValue IS NOT NULL
AND sortableValue IS NULL
AND ISJSON(textValue) = 1
AND JSON_VALUE(textValue, '$.date') IS NOT NULL";

var rowsAffected = database.Execute(sql);
logger.LogInformation(
"Populated sortableValue for {RowCount} property data rows using SQL Server JSON functions.",
rowsAffected);
UPDATE TOP ({BatchSize}) pd
SET sortableValue = CONVERT(varchar(50), SWITCHOFFSET(parsed.parsedDate, '+00:00'), 127)
FROM umbracoPropertyData pd
CROSS APPLY (SELECT TRY_CAST(JSON_VALUE(pd.textValue, '$.date') AS datetimeoffset) AS parsedDate) parsed
WHERE pd.propertyTypeId IN ({propertyTypeIdsInClause})
AND pd.textValue IS NOT NULL
AND pd.sortableValue IS NULL
AND ISJSON(pd.textValue) = 1
AND parsed.parsedDate IS NOT NULL";

return rowsAffected;
return ExecuteInBatches(database, sql, logger, "SQL Server");
}

/// <summary>
Expand All @@ -103,36 +129,54 @@ AND ISJSON(textValue) = 1
/// SQLite has limited datetime manipulation capabilities, so we extract the date string
/// and use strftime to normalize it. For dates with timezone offsets, SQLite's datetime
/// function can parse ISO 8601 format and converts to UTC.
/// The resulting format is: yyyy-MM-ddTHH:mm:ssZ
/// The resulting format is: yyyy-MM-ddTHH:mm:ssZ.
/// The datetime() IS NOT NULL guard in the WHERE clause excludes unparseable dates so the
/// batch loop always makes progress (writing NULL back would otherwise re-select the row).
/// </remarks>
private static int MigrateSQLite(IUmbracoDatabase database, string aliasesInClause, ILogger logger)
private static int MigrateSQLite(IUmbracoDatabase database, string propertyTypeIdsInClause, ILogger logger)
{
// SQLite: Use json_valid to validate JSON before parsing, then use json_extract to get the date value,
// datetime() to parse and normalize to UTC. The datetime() function automatically handles timezone
// offsets in ISO 8601 format and converts to UTC.
// We then format using strftime to get a consistent sortable format.
// SQLite's default build does not support UPDATE ... LIMIT, so constrain the update via a
// subquery that selects a batch of matching row ids.
var sql = $@"
UPDATE umbracoPropertyData
SET sortableValue = strftime('%Y-%m-%dT%H:%M:%SZ', datetime(json_extract(textValue, '$.date')))
WHERE propertyTypeId IN (
WHERE id IN (
SELECT id
FROM cmsPropertyType
WHERE dataTypeId IN (
SELECT nodeId
FROM umbracoDataType
WHERE propertyEditorAlias IN ({aliasesInClause})
)
)
AND textValue IS NOT NULL
AND sortableValue IS NULL
AND json_valid(textValue) = 1
AND json_extract(textValue, '$.date') IS NOT NULL";

var rowsAffected = database.Execute(sql);
FROM umbracoPropertyData
WHERE propertyTypeId IN ({propertyTypeIdsInClause})
AND textValue IS NOT NULL
AND sortableValue IS NULL
AND json_valid(textValue) = 1
AND datetime(json_extract(textValue, '$.date')) IS NOT NULL
LIMIT {BatchSize}
Comment thread
AndyButland marked this conversation as resolved.
)";

return ExecuteInBatches(database, sql, logger, "SQLite");
}

private static int ExecuteInBatches(IUmbracoDatabase database, string sql, ILogger logger, string databaseProviderName)
{
var totalRowsAffected = 0;
while (true)
{
var rowsAffected = database.Execute(sql);
if (rowsAffected <= 0)
{
break;
}

totalRowsAffected += rowsAffected;
logger.LogInformation(
"Populated sortableValue for batch of {BatchRowCount} property data rows ({TotalRowCount} total so far).",
rowsAffected,
totalRowsAffected);
}

logger.LogInformation(
"Populated sortableValue for {RowCount} property data rows using SQLite JSON functions.",
rowsAffected);
"Populated sortableValue for {RowCount} property data rows using {DatabaseFlavour} JSON functions.",
totalRowsAffected,
databaseProviderName);

return rowsAffected;
return totalRowsAffected;
}
}
Loading