Skip to content

Migrations: Optimise ConvertLocalLinks migration to process data in pages, to avoid having to load all property data into memory #21003

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Zeegaan merged 4 commits into from
Dec 2, 2025
Merged

Migrations: Optimise ConvertLocalLinks migration to process data in pages, to avoid having to load all property data into memory #21003

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
49e3894
Optimize ConvertLocalLinks migration to process data in pages, to avo…
AndyButland Nov 29, 2025
0363fe6
Apply suggestions from code review
AndyButland Nov 29, 2025
2f19615
Merge branch 'main' into v17/improvement/optimize-convert-local-links…
AndyButland Dec 1, 2025
7ab882d
Updated obsoletion warning.
AndyButland Dec 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
223 changes: 141 additions & 82 deletions src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/ConvertLocalLinks.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using System.Collections.Concurrent;

Check warning on line 1 in src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/ConvertLocalLinks.cs

View check run for this annotation

CodeScene Delta Analysis / CodeScene Code Health Review (main)

❌ New issue: Overall Code Complexity 

This module has a mean cyclomatic complexity of 5.29 across 7 functions. The mean complexity threshold is 4. This file has many conditional statements (e.g. if, for, while) across its implementation, leading to lower code health. Avoid adding more conditionals.

Check warning on line 1 in src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/ConvertLocalLinks.cs

View check run for this annotation

CodeScene Delta Analysis / CodeScene Code Health Review (main)

❌ New issue: Missing Arguments Abstractions 

The average number of function arguments in this module is 4.43 across 7 functions. The average arguments threshold is 4.00. The functions in this file have too many arguments, indicating a lack of encapsulation or too many responsibilities in the same functions. Avoid adding more.
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using NPoco;
Expand All @@ -17,6 +17,13 @@

namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0;

/// <summary>
/// Migrates local links in content and media properties from the legacy format using UDIs
/// to the new one with GUIDs.
/// </summary>
/// <remarks>
/// See: https://github.com/umbraco/Umbraco-CMS/pull/17307.
/// </remarks>
public class ConvertLocalLinks : MigrationBase
{
private readonly IUmbracoContextFactory _umbracoContextFactory;
Expand All @@ -30,7 +37,9 @@
private readonly ICoreScopeProvider _coreScopeProvider;
private readonly LocalLinkMigrationTracker _linkMigrationTracker;

[Obsolete("Use non obsoleted contructor instead")]
/// <summary>
/// Initializes a new instance of the <see cref="ConvertLocalLinks"/> class.
/// </summary>
public ConvertLocalLinks(
IMigrationContext context,
IUmbracoContextFactory umbracoContextFactory,
Expand All @@ -57,6 +66,10 @@
_linkMigrationTracker = linkMigrationTracker;
}

/// <summary>
/// Initializes a new instance of the <see cref="ConvertLocalLinks"/> class.
/// </summary>
[Obsolete("Please use the constructor taking all parameters. Scheduled for removal along with all other migrations to 17 in Umbraco 18.")]
public ConvertLocalLinks(
IMigrationContext context,
IUmbracoContextFactory umbracoContextFactory,
Expand All @@ -83,6 +96,7 @@
{
}

/// <inheritdoc/>
protected override void Migrate()
{
IEnumerable<string> propertyEditorAliases = _localLinkProcessor.GetSupportedPropertyEditorAliases();
Expand Down Expand Up @@ -116,7 +130,7 @@
_logger.LogInformation(
"Migration starting for all properties of type: {propertyEditorAlias}",
propertyEditorAlias);
if (ProcessPropertyTypes(propertyTypes, languagesById))
if (ProcessPropertyTypes(propertyEditorAlias, propertyTypes, languagesById))
{
_logger.LogInformation(
"Migration succeeded for all properties of type: {propertyEditorAlias}",
Expand All @@ -134,7 +148,7 @@
RebuildCache = true;
}

private bool ProcessPropertyTypes(IPropertyType[] propertyTypes, IDictionary<int, ILanguage> languagesById)
private bool ProcessPropertyTypes(string propertyEditorAlias, IPropertyType[] propertyTypes, IDictionary<int, ILanguage> languagesById)
{
foreach (IPropertyType propertyType in propertyTypes)
{
Expand All @@ -145,112 +159,157 @@
?? throw new InvalidOperationException(
"The data type value editor could not be fetched.");

Sql<ISqlContext> sql = Sql()
.Select<PropertyDataDto>()
.From<PropertyDataDto>()
.InnerJoin<ContentVersionDto>()
.On<PropertyDataDto, ContentVersionDto>((propertyData, contentVersion) =>
propertyData.VersionId == contentVersion.Id)
.LeftJoin<DocumentVersionDto>()
.On<ContentVersionDto, DocumentVersionDto>((contentVersion, documentVersion) =>
contentVersion.Id == documentVersion.Id)
.Where<PropertyDataDto, ContentVersionDto, DocumentVersionDto>(
(propertyData, contentVersion, documentVersion) =>
(contentVersion.Current == true || documentVersion.Published == true)
&& propertyData.PropertyTypeId == propertyType.Id);

List<PropertyDataDto> propertyDataDtos = Database.Fetch<PropertyDataDto>(sql);
if (propertyDataDtos.Count < 1)
long propertyDataCount = Database.ExecuteScalar<long>(BuildPropertyDataSql(propertyType, true));
if (propertyDataCount == 0)
{
continue;
}

var updateBatch = propertyDataDtos.Select(propertyDataDto =>
UpdateBatch.For(propertyDataDto, Database.StartSnapshot(propertyDataDto))).ToList();

var updatesToSkip = new ConcurrentBag<UpdateBatch<PropertyDataDto>>();

var progress = 0;
_logger.LogInformation(
"Migrating {PropertyDataCount} property data values for property {PropertyTypeAlias} ({PropertyTypeKey}) with property editor alias {PropertyEditorAlias}",
propertyDataCount,
propertyType.Alias,
propertyType.Key,
propertyEditorAlias);

void HandleUpdateBatch(UpdateBatch<PropertyDataDto> update)
// Process in pages to avoid loading all property data from the database into memory at once.
Sql<ISqlContext> sql = BuildPropertyDataSql(propertyType);
const int PageSize = 10000;
long pageNumber = 1;
long pageCount = (propertyDataCount + PageSize - 1) / PageSize;
int processedCount = 0;
while (processedCount < propertyDataCount)
{
using UmbracoContextReference umbracoContextReference = _umbracoContextFactory.EnsureUmbracoContext();

progress++;
if (progress % 100 == 0)
Page<PropertyDataDto> propertyDataDtoPage = Database.Page<PropertyDataDto>(pageNumber, PageSize, sql);
if (propertyDataDtoPage.Items.Count == 0)
{
_logger.LogInformation(" - finíshed {progress} of {total} properties", progress,
updateBatch.Count);
break;
}

PropertyDataDto propertyDataDto = update.Poco;
var updateBatchCollection = propertyDataDtoPage.Items
.Select(propertyDataDto =>
UpdateBatch.For(propertyDataDto, Database.StartSnapshot(propertyDataDto)))
.ToList();

if (ProcessPropertyDataDto(propertyDataDto, propertyType, languagesById, valueEditor) == false)
var updatesToSkip = new ConcurrentBag<UpdateBatch<PropertyDataDto>>();

var progress = 0;

void HandleUpdateBatch(UpdateBatch<PropertyDataDto> update)
{
updatesToSkip.Add(update);
using UmbracoContextReference umbracoContextReference = _umbracoContextFactory.EnsureUmbracoContext();

progress++;
if (progress % 100 == 0)
{
_logger.LogInformation(
" - finished {Progress} of {PageTotal} properties in page {PageNumber} of {PageCount}",
progress,
updateBatchCollection.Count,
pageNumber,
pageCount);
}

PropertyDataDto propertyDataDto = update.Poco;

if (ProcessPropertyDataDto(propertyDataDto, propertyType, languagesById, valueEditor) == false)
Comment thread
AndyButland marked this conversation as resolved.
{
updatesToSkip.Add(update);
}
}
}

if (DatabaseType == DatabaseType.SQLite)
{
// SQLite locks up if we run the migration in parallel, so... let's not.
foreach (UpdateBatch<PropertyDataDto> update in updateBatch)
if (DatabaseType == DatabaseType.SQLite)
{
HandleUpdateBatch(update);
// SQLite locks up if we run the migration in parallel, so... let's not.
foreach (UpdateBatch<PropertyDataDto> update in updateBatchCollection)
{
HandleUpdateBatch(update);
}
}
}
else
{
Parallel.ForEachAsync(updateBatch, async (update, token) =>
else
{
//Foreach here, but we need to suppress the flow before each task, but not the actuall await of the task
Task task;
using (ExecutionContext.SuppressFlow())
Parallel.ForEachAsync(updateBatchCollection, async (update, token) =>
{
task = Task.Run(
() =>
{
using ICoreScope scope = _coreScopeProvider.CreateCoreScope();
scope.Complete();
HandleUpdateBatch(update);
},
token);
}
//Foreach here, but we need to suppress the flow before each task, but not the actual await of the task
Task task;
using (ExecutionContext.SuppressFlow())
{
task = Task.Run(
() =>
{
using ICoreScope scope = _coreScopeProvider.CreateCoreScope();
scope.Complete();
HandleUpdateBatch(update);
},
token);
}

await task;
}).GetAwaiter().GetResult();
}

await task;
}).GetAwaiter().GetResult();
}
updateBatchCollection.RemoveAll(updatesToSkip.Contains);

updateBatch.RemoveAll(updatesToSkip.Contains);
if (updateBatchCollection.Any() is false)
{
_logger.LogDebug(" - no properties to convert, continuing");

if (updateBatch.Any() is false)
{
_logger.LogDebug(" - no properties to convert, continuing");
continue;
}
pageNumber++;
processedCount += propertyDataDtoPage.Items.Count;

_logger.LogInformation(" - {totalConverted} properties converted, saving...", updateBatch.Count);
var result = Database.UpdateBatch(updateBatch, new BatchOptions { BatchSize = 100 });
if (result != updateBatch.Count)
{
throw new InvalidOperationException(
$"The database batch update was supposed to update {updateBatch.Count} property DTO entries, but it updated {result} entries.");
}
continue;
}

_logger.LogDebug(
"Migration completed for property type: {propertyTypeName} (id: {propertyTypeId}, alias: {propertyTypeAlias}, editor alias: {propertyTypeEditorAlias}) - {updateCount} property DTO entries updated.",
propertyType.Name,
propertyType.Id,
propertyType.Alias,
propertyType.PropertyEditorAlias,
result);
_logger.LogInformation(" - {totalConverted} properties converted, saving...", updateBatchCollection.Count);
var result = Database.UpdateBatch(updateBatchCollection, new BatchOptions { BatchSize = 100 });
if (result != updateBatchCollection.Count)
{
throw new InvalidOperationException(
$"The database batch update was supposed to update {updateBatchCollection.Count} property DTO entries, but it updated {result} entries.");
}

_logger.LogDebug(
"Migration completed for property type: {propertyTypeName} (id: {propertyTypeId}, alias: {propertyTypeAlias}, editor alias: {propertyTypeEditorAlias}) - {updateCount} property DTO entries updated.",
propertyType.Name,
propertyType.Id,
propertyType.Alias,
propertyType.PropertyEditorAlias,
result);

pageNumber++;
processedCount += propertyDataDtoPage.Items.Count;
}
}

return true;
}

private bool ProcessPropertyDataDto(PropertyDataDto propertyDataDto, IPropertyType propertyType,
IDictionary<int, ILanguage> languagesById, IDataValueEditor valueEditor)
private Sql<ISqlContext> BuildPropertyDataSql(IPropertyType propertyType, bool isCount = false)
{
Sql<ISqlContext> sql = isCount
? Sql().SelectCount()
: Sql().Select<PropertyDataDto>();

sql = sql.From<PropertyDataDto>()
.InnerJoin<ContentVersionDto>()
.On<PropertyDataDto, ContentVersionDto>((propertyData, contentVersion) =>
propertyData.VersionId == contentVersion.Id)
.LeftJoin<DocumentVersionDto>()
.On<ContentVersionDto, DocumentVersionDto>((contentVersion, documentVersion) =>
contentVersion.Id == documentVersion.Id)
.Where<PropertyDataDto, ContentVersionDto, DocumentVersionDto>(
(propertyData, contentVersion, documentVersion) =>
(contentVersion.Current || documentVersion.Published)
&& propertyData.PropertyTypeId == propertyType.Id);

return sql;
}

private bool ProcessPropertyDataDto(
PropertyDataDto propertyDataDto,
IPropertyType propertyType,
IDictionary<int, ILanguage> languagesById,
IDataValueEditor valueEditor)
{
// NOTE: some old property data DTOs can have variance defined, even if the property type no longer varies
var culture = propertyType.VariesByCulture()
Expand Down
Loading