V16/new trash handling

uSync.Core/Extensions/JsonTextExtensions.cs

@@ -31,6 +31,8 @@ public static class JsonTextExtensions
             new JsonUdiRangeConverter(),
             new JsonBooleanConverter(),
             new JsonXElementConverter(),
+            new JsonBlockListLayoutItemConverter(),
+            new JsonBlockGridLayoutItemConverter()
         }
     };
 

uSync.Core/Json/JsonBlockItemConverters.cs

@@ -0,0 +1,78 @@
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+using Umbraco.Cms.Core.Models.Blocks;
+
+namespace uSync.Core.Json;
+
+/// <summary>
+/// in v16 the BlockLayoutItem(s) have obsolete properties that are sometimes set and sometimes not
+/// this leads to false positive's when looking for changes. 
+/// 
+///  these two custom converters write those properties out as null, so they never change. causing
+///  the serialized json to be consistent. 
+/// </summary>
+
+public abstract class JsonBlockItemConverterBase<T> : JsonConverter<T>
+    where T : BlockLayoutItemBase, new()
+{
+    public override T? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        if (reader.TokenType != JsonTokenType.StartObject)
+            throw new JsonException("Invalid JSON expecting start object");
+
+        var item = new T();
+
+        while (reader.Read())
+        {
+            if (reader.TokenType == JsonTokenType.EndObject)
+                return item;
+
+            if (reader.TokenType != JsonTokenType.PropertyName)
+                throw new JsonException("Expecting property name");
+
+            var propertyName = reader.GetString();
+            reader.Read();
+
+            switch (propertyName)
+            {
+                case "contentKey":
+                    var contentKey = reader.GetString();
+                    if (contentKey != null && Guid.TryParse(contentKey, out var contentGuid))
+                        item.ContentKey = contentGuid;
+                    break;
+                case "settingsKey":
+                    var settingsKey = reader.GetString();
+                    if (settingsKey != null && Guid.TryParse(settingsKey, out var settingsGuid))
+                        item.SettingsKey = settingsGuid;
+                    break;
+                default:
+                    // we don't care about the obsolete properties here...
+                    break;
+            }
+        }
+
+        throw new JsonException("Unexpected end of JSON");
+    }
+
+    public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options)
+    {
+        // we could just not write out the obsolete properties, but they will appear as null it 
+        // lots of people's existing exports , so we can write them out as null to keep things consistent.
+        writer.WriteStartObject();
+        writer.WriteString("contentKey", value.ContentKey.ToString());
+        writer.WriteNull("contentUdi");
+
+        if (value.SettingsKey.HasValue && value.SettingsKey != Guid.Empty)
+            writer.WriteString("settingsKey", value.SettingsKey.ToString());
+        else
+            writer.WriteNull("settingsKey");
+
+        writer.WriteNull("settingsUdi");
+        writer.WriteEndObject();
+    }
+}
+
+public class JsonBlockListLayoutItemConverter : JsonBlockItemConverterBase<BlockListLayoutItem> { }
+
+public class JsonBlockGridLayoutItemConverter : JsonBlockItemConverterBase<BlockGridLayoutItem> { }

uSync.Core/Json/XElementJsonConverter.cs

@@ -16,3 +16,4 @@ public override void Write(Utf8JsonWriter writer, XElement value, JsonSerializer
         writer.WriteStringValue(value.ToString());
     }
 }
+

uSync.Core/Serialization/Serializers/ContentSerializer.cs

@@ -175,19 +175,6 @@ protected override async Task<SyncAttempt<IContent>> DeserializeCoreAsync(XEleme
 
         details.AddRange(await DeserializeBaseAsync(item, node, options));
 
-        var infoNode = node.Element(uSyncConstants.Xml.Info);
-
-        if (infoNode is not null)
-        {
-            var trashed = infoNode.Element("Trashed").ValueOrDefault(false);
-            var restoreParent = infoNode.Element("Trashed")?.Attribute("Parent").ValueOrDefault(Guid.Empty) ?? Guid.Empty;
-            details.AddNotNull(HandleTrashedState(item, trashed, restoreParent));
-        }
-
-        // cultures...
-
-
-
         details.AddNotNull(await DeserializeTemplate(item, node));
 
         var propertiesAttempt = await DeserializePropertiesAsync(item, node, options);
@@ -303,11 +290,15 @@ public int DeserializeWriterInfo(IContent item, XElement node, SyncSerializerOpt
     /// </remarks>
     public override async Task<SyncAttempt<IContent>> DeserializeSecondPassAsync(IContent item, XElement node, SyncSerializerOptions options)
     {
+        var details = new List<uSyncChange>();
+
+        // move trashed state to second pass, as the item needs an Id for the relation to work. 
+        details.AddNotNull(await DeserializeTrashed(node, item, Constants.Conventions.RelationTypes.RelateParentDocumentOnDeleteAlias));
+
         // move sort to second pass, as if we attempt to set this 
         // on a brand new item, it doesn't get set. 
         // doing it on second pass ensures it gets set on the item
         // after it has been saved by umbraco. 
-        var details = new List<uSyncChange>();
         if (!options.GetSetting<bool>(
             uSyncConstants.DefaultSettings.IgnoreSortOrder,
             uSyncConstants.DefaultSettings.IgnoreSortOrder_Default))
@@ -417,41 +408,12 @@ private static ContentSchedule GetContentScheduleFromNode(XElement scheduleNode)
         return null;
     }
 
+    // trashed helpers. 
+    protected override void MoveToRecycleBin(IContent item) => contentService.MoveToRecycleBin(item);
+    protected override void SetTrashed(IContent item) => ((ContentBase)item).Trashed = true;
+    protected override void MoveItem(IContent item, int parentId) => contentService.Move(item, parentId);
+    protected override IContent? GetByKey(Guid id) => contentService.GetById(id);
 
-    protected override uSyncChange? HandleTrashedState(IContent item, bool trashed, Guid restoreParentKey)
-    {
-        if (!trashed && item.Trashed)
-        {
-            // if the item is trashed, then the change of it's parent 
-            // should restore it (as long as we do a move!)
-
-
-            var restoreParentId = GetRelationParentId(item, restoreParentKey, Constants.Conventions.RelationTypes.RelateParentDocumentOnDeleteAlias);
-            contentService.Move(item, restoreParentId);
-
-            // clean out any relations for this item (some versions of Umbraco don't do this on a Move)
-            CleanRelations(item, Constants.Conventions.RelationTypes.RelateParentDocumentOnDeleteAlias);
-
-            return uSyncChange.Update("Restored", item.Name ?? item.Id.ToString(), "Recycle Bin", restoreParentKey.ToString());
-
-        }
-        else if (trashed && !item.Trashed)
-        {
-            // not already in the recycle bin?
-            if (item.ParentId > Constants.System.RecycleBinContent)
-            {
-                // clean any relations that may be there (stops an error)
-                CleanRelations(item, Constants.Conventions.RelationTypes.RelateParentDocumentOnDeleteAlias);
-
-                // move to the recycle bin    
-                contentService.MoveToRecycleBin(item);
-            }
-
-            return uSyncChange.Update("Moved to Bin", item.Name ?? item.Id.ToString(), "", "Recycle Bin");
-        }
-
-        return null;
-    }
 
     protected virtual Attempt<string?> DoSaveOrPublish(IContent item, XElement node, SyncSerializerOptions options)
     {

uSync.Core/Serialization/Serializers/ContentSerializerBase.cs

@@ -1,4 +1,6 @@
-using Microsoft.Extensions.Logging;
+using Lucene.Net.Queries.Function.ValueSources;
+
+using Microsoft.Extensions.Logging;
 
 using System.Globalization;
 using System.Text.RegularExpressions;
@@ -649,9 +651,69 @@ private static bool IsUpdatedValue(object? current, object? newValue)
         return null;
     }
 
+    protected abstract void MoveToRecycleBin(TObject item);
+    protected abstract void SetTrashed(TObject item);
+    protected abstract void MoveItem(TObject item, int parentId);
+    protected abstract TObject? GetByKey(Guid id);
+
+    protected async Task<uSyncChange?> DeserializeTrashed(XElement node, TObject item, string relationAlias)
+    {
+        var info = node.Element(uSyncConstants.Xml.Info);
+        if (info is null) return null;
+
+        var trashed = info.Element("Trashed").ValueOrDefault(false);
+        var restoreParent = info.Element("Trashed")?.Attribute("Parent").ValueOrDefault(Guid.Empty) ?? Guid.Empty;
+        return await HandleTrashedState(item, trashed, restoreParent, relationAlias);
+    }
+
+    [Obsolete("Use HandleTrashedState with relationAlias, will be removed in v18")]
     protected virtual uSyncChange? HandleTrashedState(TObject item, bool trashed, Guid restoreParent)
         => uSyncChange.NoChange($"Member/{item.Name}", item.Name ?? item.Id.ToString());
 
+    protected virtual Task<uSyncChange?> HandleTrashedState(TObject item, bool trashed, Guid restoreParent, string relationAlias)
+    {
+        if (!trashed && item.Trashed)
+        {
+            // if the item is trashed, then the change of it's parent 
+            // should restore it (as long as we do a move!)
+
+            var restoreParentId = GetRelationParentId(item, restoreParent, relationAlias);
+            MoveItem(item, restoreParentId);
+
+            // clean out any relations for this item (some versions of Umbraco don't do this on a Move)
+            CleanRelations(item, relationAlias);
+
+            return Task.FromResult<uSyncChange?>(uSyncChange.Update("Restored", item.Name ?? item.Id.ToString(), "Recycle Bin", restoreParent.ToString()));
+
+        }
+        else if (trashed && !item.Trashed)
+        {
+            // not already in the recycle bin?
+            if (item.ParentId > Constants.System.RecycleBinContent)
+            {
+                // clean any relations that may be there (stops an error)
+                CleanRelations(item, relationAlias);
+
+                // move to the recycle bin    
+                MoveToRecycleBin(item);
+            }
+            else
+            {
+                // on first import the item might be in the recycle bin, but not marked as trash.
+                // but one does not simple set 'trashed' on a content item.
+                SetTrashed(item);
+
+                AddRelation(relationAlias, restoreParent, item.Id);
+            }
+
+            return Task.FromResult<uSyncChange?>(uSyncChange.Update("Moved to Bin", item.Name ?? item.Id.ToString(), "", "Recycle Bin"));
+        }
+
+        return Task.FromResult<uSyncChange?>(null);
+
+    }
+
+
     protected async Task<string> GetExportValueAsync(object? value, IPropertyType propertyType, string culture, string segment)
     {
         if (value is null) return string.Empty;
@@ -1012,7 +1074,19 @@ protected void CleanRelations(TObject item, string relationType)
         {
             logger.LogWarning(exception, "Error cleaning up relations: {id}", item.Id);
         }
+    }
 
+    /// <summary>
+    ///  a relation for the given item, if the relation already exists, its not added again.
+    /// </summary>
+    private void AddRelation(string relationAlias, Guid parentKey, int childId)
+    {
+        var existing = relationService.GetByChildId(childId, relationAlias);
+        if (existing.Any()) return;
+
+        var parent = GetByKey(parentKey);
+        if (parent != null && childId > 0)
+            relationService.Relate(parent.Id, childId, relationAlias);
     }
 
     protected int GetRelationParentId(TObject item, Guid restoreParentKey, string relationType)
@@ -1041,8 +1115,8 @@ protected int GetRelationParentId(TObject item, Guid restoreParentKey, string re
     private List<string> GetExcludedProperties(SyncSerializerOptions options)
     {
         List<string> exclude = [.. dontSerialize];
-        
-        var excludeOptions = options.GetSetting<string>(uSyncConstants.DefaultSettings.DoNotSerialize, 
+
+        var excludeOptions = options.GetSetting<string>(uSyncConstants.DefaultSettings.DoNotSerialize,
             uSyncConstants.DefaultSettings.DoNotSerialize_Default);
 
         if (!string.IsNullOrWhiteSpace(excludeOptions))
@@ -1065,7 +1139,7 @@ private List<string> GetExcludedProperties(SyncSerializerOptions options)
         }
         catch (ArgumentException ex)
         {
-            logger.LogDebug("Unable to parse pattern '{pattern}' from '{settingsKey}' as Regex. {error}. Pattern will not be considered.", pattern, 
+            logger.LogDebug("Unable to parse pattern '{pattern}' from '{settingsKey}' as Regex. {error}. Pattern will not be considered.", pattern,
                 uSyncConstants.DefaultSettings.DoNotSerializePattern, ex.Message);
             return null;
         }

uSync.Core/Serialization/Serializers/MediaSerializer.cs

@@ -59,19 +59,13 @@ protected override async Task<SyncAttempt<IMedia>> DeserializeCoreAsync(XElement
 
         details.AddRange(await DeserializeBaseAsync(item, node, options));
 
-        var info = node.Element(uSyncConstants.Xml.Info);
-        if (info is not null)
-        {
-            var trashed = info.Element("Trashed").ValueOrDefault(false);
-            var restoreParent = info.Element("Trashed")?.Attribute("Parent").ValueOrDefault(Guid.Empty) ?? Guid.Empty;
-            details.AddNotNull(HandleTrashedState(item, trashed, restoreParent));
-        }
-
         var propertyAttempt = await DeserializePropertiesAsync(item, node, options);
         if (!propertyAttempt.Success)
             return SyncAttempt<IMedia>.Fail(item.Name ?? item.Id.ToString(), item, ChangeType.Fail, "Failed to save properties",
                 propertyAttempt.Exception ?? new Exception($"Error with properties {item.Id}"));
 
+        var info = node.Element(uSyncConstants.Xml.Info);
+
         if (!options.GetSetting<bool>(uSyncConstants.DefaultSettings.IgnoreSortOrder, uSyncConstants.DefaultSettings.IgnoreSortOrder_Default))
         {
             var sortOrder = info?.Element(uSyncConstants.Xml.SortOrder).ValueOrDefault(-1) ?? -1;
@@ -103,33 +97,20 @@ protected override async Task<SyncAttempt<IMedia>> DeserializeCoreAsync(XElement
         return SyncAttempt<IMedia>.Succeed(item.Name ?? item.Id.ToString(), item, ChangeType.Import, "", true, propertyAttempt.Result);
     }
 
-    protected override uSyncChange? HandleTrashedState(IMedia item, bool trashed, Guid restoreParentKey)
+    public override async Task<SyncAttempt<IMedia>> DeserializeSecondPassAsync(IMedia item, XElement node, SyncSerializerOptions options)
     {
-        if (!trashed && item.Trashed)
-        {
-            // if the item is trashed, then moving it back to the parent value 
-            // restores it.
-
-            var restoreParentId = GetRelationParentId(item, restoreParentKey, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias);
-            _mediaService.Move(item, restoreParentId);
-
-            CleanRelations(item, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias);
-
-            return uSyncChange.Update("Restored", item.Name ?? item.Id.ToString(), "Recycle Bin", item.ParentId.ToString());
-        }
-        else if (trashed && !item.Trashed)
-        {
-            // clean any rouge relations 
-            CleanRelations(item, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias);
-
-            // move to the recycle bin
-            _mediaService.MoveToRecycleBin(item);
-            return uSyncChange.Update("Moved to Bin", item.Name ?? item.Id.ToString(), "", "Recycle Bin");
-        }
+        var details = new List<uSyncChange>();
+        details.AddNotNull(await DeserializeTrashed(node, item, Constants.Conventions.RelationTypes.RelateParentMediaFolderOnDeleteAlias));
 
-        return null;
+        return SyncAttempt<IMedia>.Succeed(item.Name ?? item.Id.ToString(), item,
+            details.Count == 0 ? ChangeType.NoChange : ChangeType.Import, details);
     }
 
+    protected override void MoveToRecycleBin(IMedia item) => _mediaService.MoveToRecycleBin(item);
+    protected override void SetTrashed(IMedia item) => ((ContentBase)item).Trashed = true;
+    protected override void MoveItem(IMedia item, int parentId) => _mediaService.Move(item, parentId);
+    protected override IMedia? GetByKey(Guid id) => _mediaService.GetById(id);
+
     protected override async Task<SyncAttempt<XElement>> SerializeCoreAsync(IMedia item, SyncSerializerOptions options)
     {
         var node = InitializeNode(item, item.ContentType.Alias, options);