Feature/source site polymorphism

.editorconfig

@@ -33,3 +33,14 @@ dotnet_diagnostic.CA1710.severity = none
 
 # CA1303: Do not pass literals as localized parameters
 dotnet_diagnostic.CA1303.severity = silent
+
+# CA2227: Collection properties should be read only
+dotnet_diagnostic.CA2227.severity = silent
+
+dotnet_naming_symbols.local.capitalisation = camel_case
+dotnet_naming_symbols.property.capitalisation = pascal_case
+
+
+
+# CA1052: Static holder types should be Static or NotInheritable
+dotnet_diagnostic.CA1052.severity = silent

MiniIndex.Core/CoreServiceInfo.cs

@@ -0,0 +1,16 @@
+using MiniIndex.Core.Minis;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace MiniIndex.Core
+{
+    public class CoreServiceInfo
+    {
+        public CoreServiceInfo(IEnumerable<IParser> parsers)
+        {
+            SupportedSites = parsers.Select(p => p.Site).ToList();
+        }
+
+        public IEnumerable<string> SupportedSites { get; }
+    }
+}

MiniIndex.Core/CoreServices.cs

@@ -0,0 +1,42 @@
+using System;
+using Lamar;
+using MediatR;
+using Microsoft.Extensions.DependencyInjection;
+using MiniIndex.Core.Http;
+using MiniIndex.Core.Minis;
+using MiniIndex.Core.Minis.Parsers.Thingiverse;
+
+namespace MiniIndex.Core
+{
+    public class CoreServices : ServiceRegistry
+    {
+        public CoreServices()
+        {
+            RegisterMediatrTypes();
+
+            this.AddHttpClient<ThingiverseClient>()
+                .SetHandlerLifetime(TimeSpan.FromMinutes(10))
+                .ApplyResiliencePolicies();
+        }
+
+        private void RegisterMediatrTypes()
+        {
+            Scan(scan =>
+            {
+                scan.TheCallingAssembly();
+
+                scan.ConnectImplementationsToTypesClosing(typeof(IRequestHandler<,>));
+                scan.ConnectImplementationsToTypesClosing(typeof(INotificationHandler<>));
+
+                scan.AddAllTypesOf<IParser>();
+            });
+
+            For<IMediator>()
+                .Use<Mediator>()
+                .Transient();
+
+            For<ServiceFactory>()
+              .Use(ctx => ctx.GetInstance);
+        }
+    }
+}

MiniIndex.Core/Http/HttpClientConfigurationExtensions.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Net;
+using System.Net.Http;
+using Microsoft.Extensions.DependencyInjection;
+using Polly;
+using Polly.CircuitBreaker;
+
+namespace MiniIndex.Core.Http
+{
+    public static class HttpClientConfigurationExtensions
+    {
+        public static IHttpClientBuilder ApplyResiliencePolicies(this IHttpClientBuilder builder)
+        {
+            Random jitter = new Random();
+
+            AsyncCircuitBreakerPolicy<HttpResponseMessage> circuitBreaker = Policy
+                .Handle<HttpRequestException>()
+                .OrResult<HttpResponseMessage>(x => x.StatusCode >= HttpStatusCode.InternalServerError)
+                .AdvancedCircuitBreakerAsync(
+                    failureThreshold: 0.5,
+                    samplingDuration: TimeSpan.FromSeconds(10),
+                    minimumThroughput: 6,
+                    durationOfBreak: TimeSpan.FromMinutes(1));
+
+            builder
+                .AddTransientHttpErrorPolicy(transient => transient
+                    .WaitAndRetryAsync(3, (retryCount) =>
+                        TimeSpan.FromSeconds(Math.Pow(retryCount, 2))
+                        + TimeSpan.FromMilliseconds(jitter.NextDouble() * 500d)))
+                .AddPolicyHandler(Policy.TimeoutAsync<HttpResponseMessage>(TimeSpan.FromSeconds(30)))
+                .AddPolicyHandler(circuitBreaker);
+
+            return builder;
+        }
+    }
+}

MiniIndex.Core/MiniIndex.Core.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.1</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.17" />
+    <PackageReference Include="Lamar" Version="4.0.0" />
+    <PackageReference Include="MediatR" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="3.1.0" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="3.1.0" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="3.1.0" />
+    <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="3.1.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Polly" Version="7.2.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\MiniIndex.Models\MiniIndex.Models.csproj" />
+    <ProjectReference Include="..\MiniIndex.Persistence\MiniIndex.Persistence.csproj" />
+  </ItemGroup>
+
+</Project>

MiniIndex.Core/Minis/IParser.cs

@@ -0,0 +1,15 @@
+using MiniIndex.Models;
+using System;
+using System.Threading.Tasks;
+
+namespace MiniIndex.Core.Minis
+{
+    public interface IParser
+    {
+        string Site { get; }
+
+        bool CanParse(Uri url);
+
+        Task<Mini> ParseFromUrl(Uri url);
+    }
+}

MiniIndex.Core/Minis/Parsers/Gumroad/GumroadParser.cs

@@ -0,0 +1,116 @@
+using HtmlAgilityPack;
+using MiniIndex.Core.Utilities;
+using MiniIndex.Models;
+using MiniIndex.Models.SourceSites;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace MiniIndex.Core.Minis.Parsers.Gumroad
+{
+    public class GumroadParser : IParser
+    {
+        public string Site => "Gumroad";
+
+        public bool CanParse(Uri url)
+        {
+            bool isGumroadUrl = url.Host.Replace("www.", "").Equals("gumroad.com", StringComparison.OrdinalIgnoreCase);
+
+            if (!isGumroadUrl)
+            {
+                return false;
+            }
+
+            return IsLFormatUrl(url) || IsHashFormatUrl(url);
+        }
+
+        public async Task<Mini> ParseFromUrl(Uri url)
+        {
+            if (IsHashFormatUrl(url))
+            {
+                string id = url.ToString().Split('#').Last();
+                url = new Uri($"https://www.gumroad.com/l/{id}");
+            }
+
+            HtmlWeb web = new HtmlWeb();
+            HtmlDocument htmlDoc = await web.LoadFromWebAsync(url, null, null);
+
+            HtmlNode creatorLink = htmlDoc.DocumentNode.SelectNodes("//a[@class=\"js-creator-profile-link\"]")
+                .FirstOrDefault();
+
+            string creatorUrl = creatorLink.GetAttributeValue("href", null);
+            string creatorName = new Uri(creatorUrl).AbsolutePath.Skip(1).AsString();
+
+            Dictionary<string, string> miniProperties = htmlDoc.DocumentNode.SelectNodes("//*[@itemprop]")
+                .Select(node => new
+                {
+                    property = node.GetAttributeValue("itemprop", null),
+                    value = GetNodeContent(node)
+                })
+                .Where(node => !String.IsNullOrWhiteSpace(node.property))
+                .ToDictionary(k => k.property, v => v.value);
+
+            Creator creator = new Creator
+            {
+                Name = creatorName
+            };
+            GumroadSource source = new GumroadSource(creator, creatorName);
+            creator.Sites.Add(source);
+
+            Mini mini = new Mini()
+            {
+                Creator = creator,
+                Name = miniProperties["name"],
+                Thumbnail = miniProperties["image"],
+                Link = miniProperties["url"]
+            };
+            mini.Sources.Add(new MiniSourceSite(mini, source, url));
+
+            return mini;
+        }
+
+        public string GetNodeContent(HtmlNode node)
+        {
+            switch (node.GetAttributeValue("itemprop", null))
+            {
+                case "url":
+                    return node.GetAttributeValue("href", null);
+
+                case "image":
+                    return node.GetAttributeValue("src", null);
+
+                default:
+                    string directText = node.GetDirectInnerText().Trim();
+
+                    if (!String.IsNullOrWhiteSpace(directText))
+                    {
+                        return directText;
+                    }
+
+                    foreach (HtmlNode innerNode in node.ChildNodes)
+                    {
+                        string innerText = GetNodeContent(innerNode);
+
+                        if (!String.IsNullOrWhiteSpace(innerText))
+                        {
+                            return innerText;
+                        }
+                    }
+                    return node.GetAttributeValue("content", null);
+            }
+        }
+
+        private static bool IsHashFormatUrl(Uri url)
+        {
+            return !String.IsNullOrWhiteSpace(url.Fragment)
+                            && url.Fragment.StartsWith("#");
+        }
+
+        private static bool IsLFormatUrl(Uri url)
+        {
+            return !String.IsNullOrWhiteSpace(url.LocalPath)
+                && url.LocalPath.StartsWith("/l/");
+        }
+    }
+}

MiniIndex.Core/Minis/Parsers/MyMiniFactory/MyMiniFactoryParser.cs

@@ -0,0 +1,71 @@
+using HtmlAgilityPack;
+using MiniIndex.Core.Utilities;
+using MiniIndex.Models;
+using MiniIndex.Models.SourceSites;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace MiniIndex.Core.Minis.Parsers.MyMiniFactory
+{
+    public class MyMiniFactoryParser : IParser
+    {
+        public string Site => "MyMiniFactory";
+
+        public bool CanParse(Uri url)
+        {
+            bool isMyMiniFactoryUrl = url.Host.Replace("www.", "").Equals("myminifactory.com", StringComparison.OrdinalIgnoreCase);
+
+            if (!isMyMiniFactoryUrl)
+            {
+                return false;
+            }
+
+            bool mmfFormat1 = !String.IsNullOrWhiteSpace(url.LocalPath)
+                && url.LocalPath.StartsWith("/object/");
+
+            return mmfFormat1;
+        }
+
+        public async Task<Mini> ParseFromUrl(Uri url)
+        {
+            HtmlWeb web = new HtmlWeb();
+            HtmlDocument htmlDoc = await web.LoadFromWebAsync(url, null, null);
+
+            HtmlNode creatorLink = htmlDoc.DocumentNode.SelectNodes("//a[@class=\"under-hover\"]")
+                .FirstOrDefault();
+
+            string creatorUrl = creatorLink.GetAttributeValue("href", null);
+            string creatorName = Uri.UnescapeDataString(creatorUrl.Split('/').Last());
+
+            Creator creator = new Creator
+            {
+                Name = creatorName
+            };
+            MyMiniFactorySource source = new MyMiniFactorySource(creator, creatorName);
+            creator.Sites.Add(source);
+
+            Mini mini = new Mini()
+            {
+                Creator = creator,
+                Name = htmlDoc.DocumentNode.SelectNodes("//h1").FirstOrDefault().InnerText.Trim(),
+                Thumbnail = htmlDoc.DocumentNode.SelectNodes("//meta").Where(n => n.Attributes.Any(a => a.Value == "og:image")).First()
+                    .Attributes.Where(a => a.Name == "content").First().Value,
+                Link = url.ToString()
+            };
+
+            int cost = 0;
+            HtmlNodeCollection priceNode = htmlDoc.DocumentNode.SelectNodes("//span[@class=\"price-title\"]");
+            if (priceNode != null)
+            {
+                cost = Int32.Parse(priceNode.First().InnerText.Remove(0, 1).Split(".").First());
+            }
+            mini.Cost = cost;
+
+            mini.Sources.Add(new MiniSourceSite(mini, source, url));
+
+            return mini;
+        }
+    }
+}