diff --git a/src/Umbraco.Core/Templates/HtmlLocalLinkParser.cs b/src/Umbraco.Core/Templates/HtmlLocalLinkParser.cs index 4627b48d3797..de573d23d183 100644 --- a/src/Umbraco.Core/Templates/HtmlLocalLinkParser.cs +++ b/src/Umbraco.Core/Templates/HtmlLocalLinkParser.cs @@ -118,8 +118,8 @@ private IEnumerable FindLocalLinkIds(string text) } } - // todo remove at some point? - private IEnumerable FindLegacyLocalLinkIds(string text) + [Obsolete("This is a temporary method to support legacy formats until we are sure all data has been migration. Scheduled for removal in v17")] + public IEnumerable FindLegacyLocalLinkIds(string text) { // Parse internal links MatchCollection tags = LocalLinkPattern.Matches(text); @@ -148,7 +148,8 @@ private IEnumerable FindLegacyLocalLinkIds(string text) } } - private class LocalLinkTag + [Obsolete("This is a temporary method to support legacy formats until we are sure all data has been migration. Scheduled for removal in v17")] + public class LocalLinkTag { public LocalLinkTag(int? intId, GuidUdi? udi, string tagHref) { diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs index 6c4becfc6d25..745e8fb5af80 100644 --- a/src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/UmbracoPlan.cs @@ -103,5 +103,6 @@ protected virtual void DefinePlan() To("{6C04B137-0097-4938-8C6A-276DF1A0ECA8}"); To("{9D3CE7D4-4884-41D4-98E8-302EB6CB0CF6}"); To("{37875E80-5CDD-42FF-A21A-7D4E3E23E0ED}"); + To("{42E44F9E-7262-4269-922D-7310CB48E724}"); } } diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/ConvertLocalLinks.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/ConvertLocalLinks.cs new file mode 100644 index 000000000000..10118acf7420 --- /dev/null +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/ConvertLocalLinks.cs @@ -0,0 +1,188 @@ +using Microsoft.Extensions.Logging; +using NPoco; +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Models.Editors; +using Umbraco.Cms.Core.Serialization; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Web; +using Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks; +using Umbraco.Cms.Infrastructure.Persistence; +using Umbraco.Cms.Infrastructure.Persistence.Dtos; +using Umbraco.Extensions; + +namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0; + +public class ConvertLocalLinks : MigrationBase +{ + private readonly IUmbracoContextFactory _umbracoContextFactory; + private readonly IContentTypeService _contentTypeService; + private readonly ILogger _logger; + private readonly IDataTypeService _dataTypeService; + private readonly ILanguageService _languageService; + private readonly IJsonSerializer _jsonSerializer; + private readonly LocalLinkProcessor _localLinkProcessor; + private readonly IMediaTypeService _mediaTypeService; + + public ConvertLocalLinks( + IMigrationContext context, + IUmbracoContextFactory umbracoContextFactory, + IContentTypeService contentTypeService, + ILogger logger, + IDataTypeService dataTypeService, + ILanguageService languageService, + IJsonSerializer jsonSerializer, + LocalLinkProcessor localLinkProcessor, + IMediaTypeService mediaTypeService) + : base(context) + { + _umbracoContextFactory = umbracoContextFactory; + _contentTypeService = contentTypeService; + _logger = logger; + _dataTypeService = dataTypeService; + _languageService = languageService; + _jsonSerializer = jsonSerializer; + _localLinkProcessor = localLinkProcessor; + _mediaTypeService = mediaTypeService; + } + + protected override void Migrate() + { + IEnumerable propertyEditorAliases = _localLinkProcessor.GetSupportedPropertyEditorAliases(); + + using UmbracoContextReference umbracoContextReference = _umbracoContextFactory.EnsureUmbracoContext(); + var languagesById = _languageService.GetAllAsync().GetAwaiter().GetResult() + .ToDictionary(language => language.Id); + + IEnumerable allContentTypes = _contentTypeService.GetAll(); + IEnumerable contentPropertyTypes = allContentTypes + .SelectMany(ct => ct.PropertyTypes); + + IMediaType[] allMediaTypes = _mediaTypeService.GetAll().ToArray(); + IEnumerable mediaPropertyTypes = allMediaTypes + .SelectMany(ct => ct.PropertyTypes); + + var relevantPropertyEditors = + contentPropertyTypes.Concat(mediaPropertyTypes).DistinctBy(pt => pt.Id) + .Where(pt => propertyEditorAliases.Contains(pt.PropertyEditorAlias)) + .GroupBy(pt => pt.PropertyEditorAlias) + .ToDictionary(group => group.Key, group => group.ToArray()); + + + foreach (var propertyEditorAlias in propertyEditorAliases) + { + if (relevantPropertyEditors.TryGetValue(propertyEditorAlias, out IPropertyType[]? propertyTypes) is false) + { + continue; + } + + _logger.LogInformation( + "Migration starting for all properties of type: {propertyEditorAlias}", + propertyEditorAlias); + if (ProcessPropertyTypes(propertyTypes, languagesById)) + { + _logger.LogInformation( + "Migration succeeded for all properties of type: {propertyEditorAlias}", + propertyEditorAlias); + } + else + { + _logger.LogError( + "Migration failed for one or more properties of type: {propertyEditorAlias}", + propertyEditorAlias); + } + } + } + + private bool ProcessPropertyTypes(IPropertyType[] propertyTypes, IDictionary languagesById) + { + foreach (IPropertyType propertyType in propertyTypes) + { + IDataType dataType = _dataTypeService.GetAsync(propertyType.DataTypeKey).GetAwaiter().GetResult() + ?? throw new InvalidOperationException("The data type could not be fetched."); + + IDataValueEditor valueEditor = dataType.Editor?.GetValueEditor() + ?? throw new InvalidOperationException( + "The data type value editor could not be fetched."); + + Sql sql = Sql() + .Select() + .From() + .InnerJoin() + .On((propertyData, contentVersion) => + propertyData.VersionId == contentVersion.Id) + .LeftJoin() + .On((contentVersion, documentVersion) => + contentVersion.Id == documentVersion.Id) + .Where( + (propertyData, contentVersion, documentVersion) => + (contentVersion.Current == true || documentVersion.Published == true) + && propertyData.PropertyTypeId == propertyType.Id); + + List propertyDataDtos = Database.Fetch(sql); + if (propertyDataDtos.Any() is false) + { + continue; + } + + foreach (PropertyDataDto propertyDataDto in propertyDataDtos) + { + if (ProcessPropertyDataDto(propertyDataDto, propertyType, languagesById, valueEditor)) + { + Database.Update(propertyDataDto); + } + } + } + + return true; + } + + private bool ProcessPropertyDataDto(PropertyDataDto propertyDataDto, IPropertyType propertyType, + IDictionary languagesById, IDataValueEditor valueEditor) + { + // NOTE: some old property data DTOs can have variance defined, even if the property type no longer varies + var culture = propertyType.VariesByCulture() + && propertyDataDto.LanguageId.HasValue + && languagesById.TryGetValue(propertyDataDto.LanguageId.Value, out ILanguage? language) + ? language.IsoCode + : null; + + if (culture is null && propertyType.VariesByCulture()) + { + // if we end up here, the property DTO is bound to a language that no longer exists. this is an error scenario, + // and we can't really handle it in any other way than logging; in all likelihood this is an old property version, + // and it won't cause any runtime issues + _logger.LogWarning( + " - property data with id: {propertyDataId} references a language that does not exist - language id: {languageId} (property type: {propertyTypeName}, id: {propertyTypeId}, alias: {propertyTypeAlias})", + propertyDataDto.Id, + propertyDataDto.LanguageId, + propertyType.Name, + propertyType.Id, + propertyType.Alias); + return false; + } + + var segment = propertyType.VariesBySegment() ? propertyDataDto.Segment : null; + var property = new Property(propertyType); + property.SetValue(propertyDataDto.Value, culture, segment); + var toEditorValue = valueEditor.ToEditor(property, culture, segment); + + _localLinkProcessor.ProcessToEditorValue(toEditorValue); + + var editorValue = _jsonSerializer.Serialize(toEditorValue); + var dbValue = valueEditor.FromEditor(new ContentPropertyData(editorValue, null), null); + if (dbValue is not string stringValue || stringValue.DetectIsJson() is false) + { + _logger.LogError( + " - value editor did not yield a valid JSON string as FromEditor value property data with id: {propertyDataId} (property type: {propertyTypeName}, id: {propertyTypeId}, alias: {propertyTypeAlias})", + propertyDataDto.Id, + propertyType.Name, + propertyType.Id, + propertyType.Alias); + return false; + } + + propertyDataDto.TextValue = stringValue; + return true; + } +} diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/ConvertLocalLinkComposer.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/ConvertLocalLinkComposer.cs new file mode 100644 index 000000000000..a34def894cd7 --- /dev/null +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/ConvertLocalLinkComposer.cs @@ -0,0 +1,17 @@ +using Microsoft.Extensions.DependencyInjection; +using Umbraco.Cms.Core.Composing; +using Umbraco.Cms.Core.DependencyInjection; + +namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks; + +[Obsolete("Will be removed in V18")] +public class ConvertLocalLinkComposer : IComposer +{ + public void Compose(IUmbracoBuilder builder) + { + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + } +} diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/ITypedLocalLinkProcessor.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/ITypedLocalLinkProcessor.cs new file mode 100644 index 000000000000..6d5b360afc99 --- /dev/null +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/ITypedLocalLinkProcessor.cs @@ -0,0 +1,11 @@ +namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks; + +[Obsolete("Will be removed in V18")] +public interface ITypedLocalLinkProcessor +{ + public Type PropertyEditorValueType { get; } + + public IEnumerable PropertyEditorAliases { get; } + + public Func, Func, bool> Process { get; } +} diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkBlocksProcessor.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkBlocksProcessor.cs new file mode 100644 index 000000000000..c3fd60304d7e --- /dev/null +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkBlocksProcessor.cs @@ -0,0 +1,54 @@ +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Models.Blocks; + +namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks; + +[Obsolete("Will be removed in V18")] +public abstract class LocalLinkBlocksProcessor +{ + public bool ProcessBlocks( + object? value, + Func processNested, + Func processStringValue) + { + if (value is not BlockValue blockValue) + { + return false; + } + + bool hasChanged = false; + + foreach (BlockItemData blockItemData in blockValue.ContentData) + { + foreach (BlockPropertyValue blockPropertyValue in blockItemData.Values) + { + if (processNested.Invoke(blockPropertyValue.Value)) + { + hasChanged = true; + } + } + } + + return hasChanged; + } +} + +[Obsolete("Will be removed in V18")] +public class LocalLinkBlockListProcessor : LocalLinkBlocksProcessor, ITypedLocalLinkProcessor +{ + public Type PropertyEditorValueType => typeof(BlockListValue); + + public IEnumerable PropertyEditorAliases => [Constants.PropertyEditors.Aliases.BlockList]; + + public Func, Func, bool> Process => ProcessBlocks; +} + +[Obsolete("Will be removed in V18")] +public class LocalLinkBlockGridProcessor : LocalLinkBlocksProcessor, ITypedLocalLinkProcessor +{ + public Type PropertyEditorValueType => typeof(BlockGridValue); + + public IEnumerable PropertyEditorAliases => [Constants.PropertyEditors.Aliases.BlockGrid]; + + public Func, Func, bool> Process => ProcessBlocks; +} diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkProcessor.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkProcessor.cs new file mode 100644 index 000000000000..a5a921fecc27 --- /dev/null +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkProcessor.cs @@ -0,0 +1,90 @@ +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Models; +using Umbraco.Cms.Core.Services; +using Umbraco.Cms.Core.Templates; + +namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks; + +[Obsolete("Will be removed in V18")] +public class LocalLinkProcessor +{ + private readonly HtmlLocalLinkParser _localLinkParser; + private readonly IIdKeyMap _idKeyMap; + private readonly IEnumerable _localLinkProcessors; + + public LocalLinkProcessor( + HtmlLocalLinkParser localLinkParser, + IIdKeyMap idKeyMap, + IEnumerable localLinkProcessors) + { + _localLinkParser = localLinkParser; + _idKeyMap = idKeyMap; + _localLinkProcessors = localLinkProcessors; + } + + public IEnumerable GetSupportedPropertyEditorAliases() => + _localLinkProcessors.SelectMany(p => p.PropertyEditorAliases); + + public bool ProcessToEditorValue(object? editorValue) + { + ITypedLocalLinkProcessor? processor = + _localLinkProcessors.FirstOrDefault(p => p.PropertyEditorValueType == editorValue?.GetType()); + + return processor is not null && processor.Process.Invoke(editorValue, ProcessToEditorValue, ProcessStringValue); + } + + public string ProcessStringValue(string input) + { + // find all legacy tags + var tags = _localLinkParser.FindLegacyLocalLinkIds(input).ToList(); + + foreach (HtmlLocalLinkParser.LocalLinkTag tag in tags) + { + string newTagHref; + if (tag.Udi is not null) + { + newTagHref = $" type=\"{tag.Udi.EntityType}\" " + + tag.TagHref.Replace(tag.Udi.ToString(), tag.Udi.Guid.ToString()); + } + else if (tag.IntId is not null) + { + // try to get the key and type from the int, else do nothing + (Guid Key, string EntityType)? conversionResult = CreateIntBasedKeyType(tag.IntId.Value); + if (conversionResult is null) + { + continue; + } + + newTagHref = $" type=\"{conversionResult.Value.EntityType}\" " + + tag.TagHref.Replace(tag.IntId.Value.ToString(), conversionResult.Value.Key.ToString()); + } + else + { + // tag does not contain enough information to convert + continue; + } + + input = input.Replace(tag.TagHref, newTagHref); + } + + return input; + } + + private (Guid Key, string EntityType)? CreateIntBasedKeyType(int id) + { + // very old data, best effort replacement + Attempt documentAttempt = _idKeyMap.GetKeyForId(id, UmbracoObjectTypes.Document); + if (documentAttempt.Success) + { + return (Key: documentAttempt.Result, EntityType: UmbracoObjectTypes.Document.ToString()); + } + + Attempt mediaAttempt = _idKeyMap.GetKeyForId(id, UmbracoObjectTypes.Media); + if (mediaAttempt.Success) + { + return (Key: mediaAttempt.Result, EntityType: UmbracoObjectTypes.Media.ToString()); + } + + return null; + } +} diff --git a/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkRteProcessor.cs b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkRteProcessor.cs new file mode 100644 index 000000000000..3682d9d09ecf --- /dev/null +++ b/src/Umbraco.Infrastructure/Migrations/Upgrade/V_15_0_0/LocalLinks/LocalLinkRteProcessor.cs @@ -0,0 +1,55 @@ +using Umbraco.Cms.Core; +using Umbraco.Cms.Core.Models.Blocks; + +namespace Umbraco.Cms.Infrastructure.Migrations.Upgrade.V_15_0_0.LocalLinks; + +[Obsolete("Will be removed in V18")] +public class LocalLinkRteProcessor : ITypedLocalLinkProcessor +{ + public Type PropertyEditorValueType => typeof(RichTextEditorValue); + + public IEnumerable PropertyEditorAliases => + [ + Constants.PropertyEditors.Aliases.TinyMce, Constants.PropertyEditors.Aliases.RichText + ]; + + public Func, Func, bool> Process => ProcessRichText; + + public bool ProcessRichText( + object? value, + Func processNested, + Func processStringValue) + { + if (value is not RichTextEditorValue richTextValue) + { + return false; + } + + bool hasChanged = false; + + var newMarkup = processStringValue.Invoke(richTextValue.Markup); + if (newMarkup.Equals(richTextValue.Markup) == false) + { + hasChanged = true; + richTextValue.Markup = newMarkup; + } + + if (richTextValue.Blocks is null) + { + return hasChanged; + } + + foreach (BlockItemData blockItemData in richTextValue.Blocks.ContentData) + { + foreach (BlockPropertyValue blockPropertyValue in blockItemData.Values) + { + if (processNested.Invoke(blockPropertyValue.Value)) + { + hasChanged = true; + } + } + } + + return hasChanged; + } +} diff --git a/src/Umbraco.Infrastructure/Persistence/NPocoSqlExtensions.cs b/src/Umbraco.Infrastructure/Persistence/NPocoSqlExtensions.cs index 0ef043509062..820657006169 100644 --- a/src/Umbraco.Infrastructure/Persistence/NPocoSqlExtensions.cs +++ b/src/Umbraco.Infrastructure/Persistence/NPocoSqlExtensions.cs @@ -45,6 +45,24 @@ public static Sql Where(this Sql sql, Ex return sql.Where(s, a); } + /// + /// Appends a WHERE clause to the Sql statement. + /// + /// The type of Dto 1. + /// The type of Dto 2. + /// The type of Dto 3. + /// The Sql statement. + /// A predicate to transform and append to the Sql statement. + /// An optional alias for Dto 1 table. + /// An optional alias for Dto 2 table. + /// An optional alias for Dto 3 table. + /// The Sql statement. + public static Sql Where(this Sql sql, Expression> predicate, string? alias1 = null, string? alias2 = null, string? alias3 = null) + { + var (s, a) = sql.SqlContext.VisitDto(predicate, alias1, alias2, alias3); + return sql.Where(s, a); + } + /// /// Appends a WHERE IN clause to the Sql statement. /// diff --git a/src/Umbraco.Infrastructure/Persistence/SqlContextExtensions.cs b/src/Umbraco.Infrastructure/Persistence/SqlContextExtensions.cs index e46963d73802..894c50109332 100644 --- a/src/Umbraco.Infrastructure/Persistence/SqlContextExtensions.cs +++ b/src/Umbraco.Infrastructure/Persistence/SqlContextExtensions.cs @@ -57,6 +57,26 @@ public static (string Sql, object[] Args) VisitDto(this ISqlContex return (visited, visitor.GetSqlParameters()); } + /// + /// Visit an expression. + /// + /// The type of the first DTO. + /// The type of the second DTO. + /// The type of the third DTO. + /// The type returned by the expression. + /// An . + /// An expression to visit. + /// An optional table alias for the first DTO. + /// An optional table alias for the second DTO. + /// An optional table alias for the third DTO. + /// A SQL statement, and arguments, corresponding to the expression. + public static (string Sql, object[] Args) VisitDto(this ISqlContext sqlContext, Expression> expression, string? alias1 = null, string? alias2 = null, string? alias3 = null) + { + var visitor = new PocoToSqlExpressionVisitor(sqlContext, alias1, alias2, alias3); + var visited = visitor.Visit(expression); + return (visited, visitor.GetSqlParameters()); + } + /// /// Visit an expression. ///