Feature/source site polymorphism

.editorconfig

@@ -33,3 +33,14 @@ dotnet_diagnostic.CA1710.severity = none
 
 # CA1303: Do not pass literals as localized parameters
 dotnet_diagnostic.CA1303.severity = silent
+
+# CA2227: Collection properties should be read only
+dotnet_diagnostic.CA2227.severity = silent
+
+dotnet_naming_symbols.local.capitalisation = camel_case
+dotnet_naming_symbols.property.capitalisation = pascal_case
+
+
+
+# CA1052: Static holder types should be Static or NotInheritable
+dotnet_diagnostic.CA1052.severity = silent

MiniIndex.Core/CoreServiceInfo.cs

@@ -0,0 +1,16 @@
+using MiniIndex.Core.Minis;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace MiniIndex.Core
+{
+    public class CoreServiceInfo
+    {
+        public CoreServiceInfo(IEnumerable<IParser> parsers)
+        {
+            SupportedSites = parsers.Select(p => p.Site).ToList();
+        }
+
+        public IEnumerable<string> SupportedSites { get; }
+    }
+}

MiniIndex.Core/CoreServices.cs

@@ -0,0 +1,42 @@
+using System;
+using Lamar;
+using MediatR;
+using Microsoft.Extensions.DependencyInjection;
+using MiniIndex.Core.Http;
+using MiniIndex.Core.Minis;
+using MiniIndex.Core.Minis.Parsers.Thingiverse;
+
+namespace MiniIndex.Core
+{
+    public class CoreServices : ServiceRegistry
+    {
+        public CoreServices()
+        {
+            RegisterMediatrTypes();
+
+            this.AddHttpClient<ThingiverseClient>()
+                .SetHandlerLifetime(TimeSpan.FromMinutes(10))
+                .ApplyResiliencePolicies();
+        }
+
+        private void RegisterMediatrTypes()
+        {
+            Scan(scan =>
+            {
+                scan.TheCallingAssembly();
+
+                scan.ConnectImplementationsToTypesClosing(typeof(IRequestHandler<,>));
+                scan.ConnectImplementationsToTypesClosing(typeof(INotificationHandler<>));
+
+                scan.AddAllTypesOf<IParser>();
+            });
+
+            For<IMediator>()
+                .Use<Mediator>()
+                .Transient();
+
+            For<ServiceFactory>()
+              .Use(ctx => ctx.GetInstance);
+        }
+    }
+}

MiniIndex.Core/Http/HttpClientConfigurationExtensions.cs

@@ -0,0 +1,36 @@
+using System;
+using System.Net;
+using System.Net.Http;
+using Microsoft.Extensions.DependencyInjection;
+using Polly;
+using Polly.CircuitBreaker;
+
+namespace MiniIndex.Core.Http
+{
+    public static class HttpClientConfigurationExtensions
+    {
+        public static IHttpClientBuilder ApplyResiliencePolicies(this IHttpClientBuilder builder)
+        {
+            Random jitter = new Random();
+
+            AsyncCircuitBreakerPolicy<HttpResponseMessage> circuitBreaker = Policy
+                .Handle<HttpRequestException>()
+                .OrResult<HttpResponseMessage>(x => x.StatusCode >= HttpStatusCode.InternalServerError)
+                .AdvancedCircuitBreakerAsync(
+                    failureThreshold: 0.5,
+                    samplingDuration: TimeSpan.FromSeconds(10),
+                    minimumThroughput: 6,
+                    durationOfBreak: TimeSpan.FromMinutes(1));
+
+            builder
+                .AddTransientHttpErrorPolicy(transient => transient
+                    .WaitAndRetryAsync(3, (retryCount) =>
+                        TimeSpan.FromSeconds(Math.Pow(retryCount, 2))
+                        + TimeSpan.FromMilliseconds(jitter.NextDouble() * 500d)))
+                .AddPolicyHandler(Policy.TimeoutAsync<HttpResponseMessage>(TimeSpan.FromSeconds(30)))
+                .AddPolicyHandler(circuitBreaker);
+
+            return builder;
+        }
+    }
+}

MiniIndex.Core/MiniIndex.Core.csproj

@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netstandard2.1</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.17" />
+    <PackageReference Include="Lamar" Version="4.0.0" />
+    <PackageReference Include="MediatR" Version="7.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Abstractions" Version="3.1.0" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="3.1.0" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="3.1.0" />
+    <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="3.1.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Polly" Version="7.2.0" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\MiniIndex.Models\MiniIndex.Models.csproj" />
+    <ProjectReference Include="..\MiniIndex.Persistence\MiniIndex.Persistence.csproj" />
+  </ItemGroup>
+
+</Project>

MiniIndex.Core/Minis/IParser.cs

@@ -0,0 +1,15 @@
+using MiniIndex.Models;
+using System;
+using System.Threading.Tasks;
+
+namespace MiniIndex.Core.Minis
+{
+    public interface IParser
+    {
+        string Site { get; }
+
+        bool CanParse(Uri url);
+
+        Task<Mini> ParseFromUrl(Uri url);
+    }
+}

MiniIndex.Core/Minis/Parsers/Gumroad/GumroadParser.cs

@@ -0,0 +1,104 @@
+using HtmlAgilityPack;
+using MiniIndex.Core.Utilities;
+using MiniIndex.Models;
+using MiniIndex.Models.SourceSites;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace MiniIndex.Core.Minis.Parsers.Gumroad
+{
+    public class GumroadParser : IParser
+    {
+        public string Site => "Gumroad";
+
+        public bool CanParse(Uri url)
+        {
+            bool isGumroadUrl = url.Host.Replace("www.", "").Equals("gumroad.com", StringComparison.OrdinalIgnoreCase);
+
+            if (!isGumroadUrl)
+            {
+                return false;
+            }
+
+            bool gumroadFormat1 = !String.IsNullOrWhiteSpace(url.LocalPath)
+                && url.LocalPath.StartsWith("/l/");
+
+            bool gumroadFormat2 = !String.IsNullOrWhiteSpace(url.Fragment)
+                && url.Fragment.StartsWith("#");
+
+            return gumroadFormat1 || gumroadFormat2;
+        }
+
+        public async Task<Mini> ParseFromUrl(Uri url)
+        {
+            HtmlWeb web = new HtmlWeb();
+            HtmlDocument htmlDoc = await web.LoadFromWebAsync(url, null, null);
+
+            HtmlNode creatorLink = htmlDoc.DocumentNode.SelectNodes("//a[@class=\"js-creator-profile-link\"]")
+                .FirstOrDefault();
+
+            string creatorUrl = creatorLink.GetAttributeValue("href", null);
+            string creatorName = new Uri(creatorUrl).AbsolutePath.Skip(1).AsString();
+
+            Dictionary<string, string> miniProperties = htmlDoc.DocumentNode.SelectNodes("//*[@itemprop]")
+                .Select(node => new
+                {
+                    property = node.GetAttributeValue("itemprop", null),
+                    value = GetNodeContent(node)
+                })
+                .Where(node => !String.IsNullOrWhiteSpace(node.property))
+                .ToDictionary(k => k.property, v => v.value);
+
+            Creator creator = new Creator
+            {
+                Name = creatorName
+            };
+            GumroadSource source = new GumroadSource(creator, creatorName);
+            creator.Sites.Add(source);
+
+            Mini mini = new Mini()
+            {
+                Creator = creator,
+                Name = miniProperties["name"],
+                Thumbnail = miniProperties["image"],
+                Link = miniProperties["url"]
+            };
+            mini.Sources.Add(new MiniSourceSite(mini, source, url));
+
+            return mini;
+        }
+
+        public string GetNodeContent(HtmlNode node)
+        {
+            switch (node.GetAttributeValue("itemprop", null))
+            {
+                case "url":
+                    return node.GetAttributeValue("href", null);
+
+                case "image":
+                    return node.GetAttributeValue("src", null);
+
+                default:
+                    string directText = node.GetDirectInnerText().Trim();
+
+                    if (!String.IsNullOrWhiteSpace(directText))
+                    {
+                        return directText;
+                    }
+
+                    foreach (HtmlNode innerNode in node.ChildNodes)
+                    {
+                        string innerText = GetNodeContent(innerNode);
+
+                        if (!String.IsNullOrWhiteSpace(innerText))
+                        {
+                            return innerText;
+                        }
+                    }
+                    return node.GetAttributeValue("content", null);
+            }
+        }
+    }
+}

MiniIndex.Core/Minis/Parsers/Shapeways/ShapewaysParser.cs

@@ -0,0 +1,62 @@
+using HtmlAgilityPack;
+using MiniIndex.Models;
+using MiniIndex.Models.SourceSites;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+
+namespace MiniIndex.Core.Minis.Parsers.Shapeways
+{
+    public class ShapewaysParser : IParser
+    {
+        public ShapewaysParser()
+        {
+        }
+
+        public string Site => "Shapeways";
+
+        public bool CanParse(Uri url)
+        {
+            return url.Host.Replace("www.", "").Equals("shapeways.com", StringComparison.OrdinalIgnoreCase);
+        }
+
+        public async Task<Mini> ParseFromUrl(Uri url)
+        {
+            HtmlWeb web = new HtmlWeb();
+            HtmlDocument htmlDoc = await web.LoadFromWebAsync(url, null, null);
+
+            string creatorName = htmlDoc.DocumentNode
+                .SelectNodes("//a[@data-sw-tracking-link-id=\"view-profile\"]")
+                .FirstOrDefault()
+                .GetAttributeValue("data-sw-tracking-target-entity-id", null);
+
+            Dictionary<string, string> miniProperties = htmlDoc.DocumentNode.SelectNodes("//div[@itemtype=\"http://schema.org/Product\"]/meta")
+                .Select(node => new
+                {
+                    property = node.GetAttributeValue("itemprop", null),
+                    content = node.GetAttributeValue("content", null)
+                })
+                .Where(node => !String.IsNullOrWhiteSpace(node.property))
+                .ToDictionary(k => k.property, v => v.content);
+
+            Creator creator = new Creator
+            {
+                Name = creatorName
+            };
+            ShapewaysSource source = new ShapewaysSource(creator, creatorName);
+            creator.Sites.Add(source);
+
+            Mini mini = new Mini()
+            {
+                Creator = creator,
+                Name = miniProperties["name"],
+                Thumbnail = miniProperties["image"],
+                Link = miniProperties["url"]
+            };
+            mini.Sources.Add(new MiniSourceSite(mini, source, url));
+
+            return mini;
+        }
+    }
+}

MiniIndex.Core/Minis/Parsers/Thingiverse/ThingiverseClient.cs

@@ -0,0 +1,41 @@
+using System;
+using System.IO;
+using System.Net.Http;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Configuration;
+using Newtonsoft.Json;
+
+namespace MiniIndex.Core.Minis.Parsers.Thingiverse
+{
+    public class ThingiverseClient
+    {
+        public ThingiverseClient(HttpClient httpClient, IConfiguration configuration)
+        {
+            httpClient.BaseAddress = new Uri("https://api.thingiverse.com");
+            _httpClient = httpClient;
+
+            _apiKey = configuration["ThingiverseToken"];
+        }
+
+        private readonly HttpClient _httpClient;
+        private readonly string _apiKey;
+
+        public async Task<ThingiverseModel.Thing> GetThing(string thingId)
+        {
+            HttpResponseMessage response = await _httpClient.GetAsync($"things/{thingId}/?access_token={_apiKey}");
+            HttpContent responseContent = response.Content;
+            if (response.StatusCode == System.Net.HttpStatusCode.OK)
+            {
+                using (StreamReader reader = new StreamReader(await responseContent.ReadAsStreamAsync()))
+                {
+                    string result = await reader.ReadToEndAsync();
+                    ThingiverseModel.Thing thing = JsonConvert.DeserializeObject<ThingiverseModel.Thing>(result);
+
+                    return thing;
+                }
+            }
+
+            return null;
+        }
+    }
+}