From b7412391947262738a98e8f2db7459217a5b9606 Mon Sep 17 00:00:00 2001 From: pcjones Date: Wed, 14 Feb 2024 23:59:53 +0100 Subject: [PATCH] Lidarr optimizations --- .../Controllers/SearchController.cs | 2 +- UmlautAdaptarr/Models/SearchItem.cs | 30 +++++++++---- UmlautAdaptarr/Providers/LidarrClient.cs | 23 +++++----- UmlautAdaptarr/Utilities/Extensions.cs | 42 ++++++++++++++++--- 4 files changed, 73 insertions(+), 24 deletions(-) diff --git a/UmlautAdaptarr/Controllers/SearchController.cs b/UmlautAdaptarr/Controllers/SearchController.cs index a12e175..564a196 100644 --- a/UmlautAdaptarr/Controllers/SearchController.cs +++ b/UmlautAdaptarr/Controllers/SearchController.cs @@ -184,7 +184,7 @@ public async Task GenericSearch([FromRoute] string options, [From if (categories.Split(',').Any(category => AUDIO_CATEGORY_IDS.Contains(category))) { var mediaType = "audio"; - searchItem = await searchItemLookupService.GetOrFetchSearchItemByExternalId(mediaType, title.ToLower()); + searchItem = await searchItemLookupService.GetOrFetchSearchItemByExternalId(mediaType, title.GetLidarrTitleForExternalId()); } } } diff --git a/UmlautAdaptarr/Models/SearchItem.cs b/UmlautAdaptarr/Models/SearchItem.cs index be8e3fb..029c790 100644 --- a/UmlautAdaptarr/Models/SearchItem.cs +++ b/UmlautAdaptarr/Models/SearchItem.cs @@ -12,6 +12,7 @@ public partial class SearchItem public bool HasUmlaut => Title?.HasUmlauts() ?? false; public string ExpectedTitle { get; set; } public string? ExpectedAuthor { get; set; } + // TODO rename GermanTitle into Foreign or LocalTitle? public string? GermanTitle { get; set; } public string[] TitleSearchVariations { get; set; } public string[] TitleMatchVariations { get; set; } @@ -42,6 +43,11 @@ public SearchItem( if (expectedTitle.Contains(expectedAuthor)) { var titleWithoutAuthorName = expectedTitle.Replace(expectedAuthor, string.Empty).RemoveExtraWhitespaces().Trim(); + + if (titleWithoutAuthorName.Length < 2) + { + // TODO log warning that this album can't be searched for automatically + } TitleMatchVariations = GenerateVariations(titleWithoutAuthorName, mediaType).ToArray(); } else @@ -86,13 +92,18 @@ public SearchItem( } } - private IEnumerable GenerateVariations(string? germanTitle, string mediaType) + private IEnumerable GenerateVariations(string? title, string mediaType) { - if (germanTitle == null) + if (title == null) + { + return []; + } + var cleanTitle = title.GetCleanTitle(); + + if (cleanTitle?.Length == 0) { return []; } - var cleanTitle = germanTitle.RemoveAccentButKeepGermanUmlauts().GetCleanTitle(); // Start with base variations including handling umlauts var baseVariations = new List @@ -121,12 +132,17 @@ private IEnumerable GenerateVariations(string? germanTitle, string media }); } - // If a german title starts with der/die/das also accept variations without it - if (mediaType != "audio" && cleanTitle.StartsWith("Der") || cleanTitle.StartsWith("Die") || cleanTitle.StartsWith("Das")) + // If a title starts with der/die/das also accept variations without it + // Same for english the, an, a + if (cleanTitle.StartsWith("Der ") || cleanTitle.StartsWith("Die ") || cleanTitle.StartsWith("Das ") + || cleanTitle.StartsWith("The ") || cleanTitle.StartsWith("An ")) { - var cleanTitleWithoutArticle = germanTitle[3..].Trim(); + var cleanTitleWithoutArticle = title[3..].Trim(); baseVariations.AddRange(GenerateVariations(cleanTitleWithoutArticle, mediaType)); - } + } else if (cleanTitle.StartsWith("A ")) + { + var cleanTitleWithoutArticle = title[2..].Trim(); + } // Remove multiple spaces var cleanedVariations = baseVariations.Select(variation => variation.RemoveExtraWhitespaces()); diff --git a/UmlautAdaptarr/Providers/LidarrClient.cs b/UmlautAdaptarr/Providers/LidarrClient.cs index 0e88778..eb02df7 100644 --- a/UmlautAdaptarr/Providers/LidarrClient.cs +++ b/UmlautAdaptarr/Providers/LidarrClient.cs @@ -42,16 +42,17 @@ public override async Task> FetchAllItemsAsync() var lidarrAlbumUrl = $"{_lidarrHost}/api/v1/album?artistId={artistId}&apikey={_lidarrApiKey}"; - if (cache.TryGetValue(lidarrAlbumUrl, out List? albums)) - { - logger.LogInformation($"Using cached albums for {UrlUtilities.RedactApiKey(lidarrAlbumUrl)}"); - } - else - { - logger.LogInformation($"Fetching all albums from artistId {artistId} from Lidarr: {UrlUtilities.RedactApiKey(lidarrArtistsUrl)}"); - var albumApiResponse = await httpClient.GetStringAsync(lidarrAlbumUrl); - albums = JsonConvert.DeserializeObject>(albumApiResponse); - } + // Disable cache for now as it can result in problems when adding new albums that aren't displayed on the artists page initially + //if (cache.TryGetValue(lidarrAlbumUrl, out List? albums)) + //{ + // logger.LogInformation($"Using cached albums for {UrlUtilities.RedactApiKey(lidarrAlbumUrl)}"); + //} + //else + //{ + logger.LogInformation($"Fetching all albums from artistId {artistId} from Lidarr: {UrlUtilities.RedactApiKey(lidarrArtistsUrl)}"); + var albumApiResponse = await httpClient.GetStringAsync(lidarrAlbumUrl); + var albums = JsonConvert.DeserializeObject>(albumApiResponse); + //} if (albums == null) { @@ -74,7 +75,7 @@ public override async Task> FetchAllItemsAsync() string[]? aliases = null; // Abuse externalId to set the search string Lidarr uses - var externalId = expectedTitle.RemoveGermanUmlautDots().RemoveAccent().RemoveSpecialCharacters().RemoveExtraWhitespaces().ToLower(); + var externalId = expectedTitle.GetLidarrTitleForExternalId(); var searchItem = new SearchItem ( diff --git a/UmlautAdaptarr/Utilities/Extensions.cs b/UmlautAdaptarr/Utilities/Extensions.cs index 0716456..8e58052 100644 --- a/UmlautAdaptarr/Utilities/Extensions.cs +++ b/UmlautAdaptarr/Utilities/Extensions.cs @@ -47,20 +47,46 @@ public static string RemoveAccentButKeepGermanUmlauts(this string text) return stringBuilder.ToString().Normalize(NormalizationForm.FormC); } - // TODO possibly replace GetCleanTitle with RemoveSpecialCharacters + public static string GetLidarrTitleForExternalId(this string text) + { + text = text.RemoveGermanUmlautDots() + .GetCleanTitle() + .ToLower(); + + // Lidarr removes the, an and a + return TitlePrefixRegex() + .Replace(text, "") + .RemoveExtraWhitespaces() + .Trim(); + } + public static string GetCleanTitle(this string text) { - return text.Replace("(", "").Replace(")", "").Replace("?","").Replace(":", "").Replace("'", ""); + return text + .Replace(".", " ") + .Replace(":", " ") + .RemoveAccentButKeepGermanUmlauts() + .RemoveSpecialCharacters(removeUmlauts: false) + .RemoveExtraWhitespaces() + .Trim(); } public static string NormalizeForComparison(this string text) { + // TODO see if we can replace RemoveGermanUmlautDots() with RemoveSpecialCharacters(removeUmlauts: false); return text.RemoveGermanUmlautDots().RemoveAccent().RemoveSpecialCharacters().Replace(" ", "").Trim().ToLower(); } - public static string RemoveSpecialCharacters(this string text) + public static string RemoveSpecialCharacters(this string text, bool removeUmlauts = true) { - return SpecialCharactersRegex().Replace(text, ""); + if (removeUmlauts) + { + return NoSpecialCharactersRegex().Replace(text, ""); + } + else + { + return NoSpecialCharactersExceptUmlautsRegex().Replace(text, ""); + } } @@ -101,9 +127,15 @@ public static bool HasUmlauts(this string text) } [GeneratedRegex("[^a-zA-Z0-9 ]+", RegexOptions.Compiled)] - private static partial Regex SpecialCharactersRegex(); + private static partial Regex NoSpecialCharactersRegex(); + + [GeneratedRegex("[^a-zA-Z0-9 öäüßÖÄÜ]+", RegexOptions.Compiled)] + private static partial Regex NoSpecialCharactersExceptUmlautsRegex(); [GeneratedRegex(@"\s+")] private static partial Regex MultipleWhitespaceRegex(); + + [GeneratedRegex(@"\b(the|an|a)\b", RegexOptions.IgnoreCase, "de-DE")] + private static partial Regex TitlePrefixRegex(); } }