feat: add cache check endpoint, require auth for /torrents endpoints

docs/Writerside/snippets/default-settings.json

@@ -18,7 +18,8 @@
       "ConnectionString": "Host=postgres;Database=zilean;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=30;CommandTimeout=3600;"
     },
     "Torrents": {
-      "EnableEndpoint": false
+      "EnableEndpoint": false,
+      "MaxHashesToCheck": 100
     },
     "Imdb": {
       "EnableImportMatching": true,

docs/Writerside/snippets/settings-with-ingestion.json

@@ -18,7 +18,8 @@
       "ConnectionString": "Host=localhost;Database=zilean;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=30;CommandTimeout=3600;"
     },
     "Torrents": {
-      "EnableEndpoint": true
+      "EnableEndpoint": true,
+      "MaxHashesToCheck": 100
     },
     "Imdb": {
       "EnableImportMatching": true,
@@ -32,7 +33,8 @@
       }],
       "ZileanInstances": [{
         "Url": "http://other-zilean:8181",
-        "EndpointType": 0
+        "EndpointType": 0,
+        "ApiKey": "SomeApiKey"
       }],
       "EnableScraping": true,
       "Kubernetes": {

docs/Writerside/topics/Api.md

@@ -71,9 +71,10 @@ This `ApiKey` is generated on first run of %product%, and stored in the settings
 > This endpoint is disabled by default, and can be enabled in the settings file see [](Configuration.md).
 {style="note"}
 
-| Path                | Description                          | Authenticated |
-|---------------------|--------------------------------------|---------------|
-| `/torrents/all`     | Stream all torrents in the database. | No            |
+| Path                    | Description                                          | Authenticated |
+|-------------------------|------------------------------------------------------|---------------|
+| `/torrents/all`         | Stream all torrents in the database.                 | Yes           |
+| `/torrents/checkcached` | Check if a hash, or collection of hashes, is cached. | Yes           |
 
 ### Healthcheck Endpoints
 

docs/Writerside/topics/Configuration.md

@@ -83,6 +83,10 @@ The database connection string comprises of the following:
 _Indicates whether the Torrents API allowing other apps to scrape the Zilean database is enabled._
 _Default: `false`_
 
+**Torrents.MaxHashesToCheck**
+_The maximum number of hashes to check in a single request to the Torrents CheckCached API._
+_Default: `100`_
+
 ### IMDB Configuration
 **Imdb.EnableImportMatching**
 _Indicates whether the indexer should import match titles to IMDB Ids during importing._
@@ -151,7 +155,7 @@ _Default: `5000`_
 
 To enable ingestion, set the `Ingestion.EnableScraping` key to `true` in the configuration.
 Also ensure that the `Ingestion.ZurgInstances` and or `Ingestion.ZileanInstances` keys are populated with the appropriate `Url`, and `EndpointType` values.
-`EndpointType` can be either `1` (Zurg) or `0` (Zilean).
+`EndpointType` can be either `1` (Zurg) or `0` (Zilean). Zilean also requires an `ApiKey` to be set.
 You do not have to specify both, you can specify one or the other, or both, depending on your requirements.
 Also there is no limit to the number of instances you can scrape from.
 

src/Zilean.ApiService/Features/Torrents/CachedItem.cs

@@ -0,0 +1,11 @@
+namespace Zilean.ApiService.Features.Torrents;
+
+public class CachedItem
+{
+    [JsonPropertyName("info_hash")]
+    public string? InfoHash { get; set; }
+    [JsonPropertyName("is_cached")]
+    public bool? IsCached { get; set; }
+    [JsonPropertyName("item")]
+    public TorrentInfo? Item { get; set; }
+}

src/Zilean.ApiService/Features/Torrents/CheckCachedRequest.cs

@@ -0,0 +1,6 @@
+namespace Zilean.ApiService.Features.Torrents;
+
+public class CheckCachedRequest
+{
+    public string? Hashes { get; set; }
+}

src/Zilean.ApiService/Features/Torrents/ErrorResponse.cs

@@ -0,0 +1,7 @@
+namespace Zilean.ApiService.Features.Torrents;
+
+public class ErrorResponse(string message)
+{
+    [JsonPropertyName("message")]
+    public string Message { get; } = message;
+}

src/Zilean.ApiService/Features/Torrents/TorrentsEndpoints.cs

@@ -4,6 +4,9 @@ public static class TorrentsEndpoints
 {
     private const string GroupName = "torrents";
     private const string Scrape = "/all";
+    private const string CheckCached = "/checkcached";
+    private const string NoHashesProvidedError = "No hashes provided";
+    private const string TooManyHashesError = "Too many hashes provided. The limit is {0}.";
 
     public static WebApplication MapTorrentsEndpoints(this WebApplication app, ZileanConfiguration configuration)
     {
@@ -13,19 +16,76 @@ public static WebApplication MapTorrentsEndpoints(this WebApplication app, Zilea
                 .WithTags(GroupName)
                 .Torrents()
                 .DisableAntiforgery()
-                .AllowAnonymous();
+                .RequireAuthorization(ApiKeyAuthentication.Policy)
+                .WithMetadata(new OpenApiSecurityMetadata(ApiKeyAuthentication.Scheme));
         }
 
         return app;
     }
 
     private static RouteGroupBuilder Torrents(this RouteGroupBuilder group)
     {
-        group.MapGet(Scrape, StreamTorrents);
+        group.MapGet(Scrape, StreamTorrents)
+            .Produces<StreamedEntry[]>()
+            .AllowAnonymous();
+
+        group.MapGet(CheckCached, CheckCachedTorrents)
+            .Produces<CachedItem[]>()
+            .ProducesProblem(StatusCodes.Status500InternalServerError)
+            .Produces<BadRequest<ErrorResponse>>();
 
         return group;
     }
 
+    private static async Task<IResult> CheckCachedTorrents(HttpContext context, ZileanDbContext dbContext, ILogger<CheckCachedRequest> logger, ZileanConfiguration configuration, [AsParameters] CheckCachedRequest request)
+    {
+        try
+        {
+            if (request.Hashes.IsNullOrWhiteSpace())
+            {
+                context.Response.StatusCode = StatusCodes.Status400BadRequest;
+                return Results.BadRequest(new ErrorResponse(NoHashesProvidedError));
+            }
+
+            var hashes = request.Hashes.Split(',');
+            if (hashes.Length >= configuration.Torrents.MaxHashesToCheck)
+            {
+                context.Response.StatusCode = StatusCodes.Status400BadRequest;
+                return Results.BadRequest(new ErrorResponse(string.Format(TooManyHashesError, configuration.Torrents.MaxHashesToCheck)));
+            }
+
+            var hashSet = new HashSet<string>(hashes);
+
+            var items = await dbContext
+                .Torrents
+                .Where(record => hashSet.Contains(record.InfoHash))
+                .Select(record => new CachedItem
+                {
+                    InfoHash = record.InfoHash,
+                    IsCached = true,
+                    Item = record
+                })
+                .ToListAsync();
+
+            foreach (var hash in hashSet.Where(hash => items.All(x => !x.InfoHash.Equals(hash, StringComparison.OrdinalIgnoreCase))))
+            {
+                items.Add(new CachedItem
+                {
+                    InfoHash = hash,
+                    IsCached = false,
+                    Item = null
+                });
+            }
+
+            return Results.Ok(items);
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Error while checking for cached availability");
+            return Results.Problem(ex.Message);
+        }
+    }
+
     private static async Task StreamTorrents(HttpContext context, ZileanDbContext dbContext, ILogger<StreamLogger> logger)
     {
         var sw = Stopwatch.StartNew();
@@ -74,4 +134,5 @@ private static async Task StreamTorrents(HttpContext context, ZileanDbContext db
     }
 
     private abstract class StreamLogger;
+    private abstract class CheckCachedLogger;
 }

src/Zilean.ApiService/GlobalUsings.cs

@@ -9,6 +9,7 @@
 global using System.Security.Claims;
 global using System.Text.Encodings.Web;
 global using System.Text.Json;
+global using System.Text.Json.Serialization;
 global using System.Xml.Serialization;
 global using Coravel;
 global using Coravel.Invocable;

src/Zilean.ApiService/Properties/launchSettings.json

@@ -13,8 +13,8 @@
         "Zilean__Ingestion__EnableScraping": "true",
         "Zilean__Dmm__EnableScraping": "true",
         "Zilean__EnableDashboard": "true",
-        "ZILEAN__NEW__API__KEY": "false",
-        "Zilean__Database__ConnectionString": "Host=localhost;Database=zilean;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=30;CommandTimeout=3600;"
+        "ZILEAN__ApiKey": "test-123",
+        "Zilean__Database__ConnectionString": "Host=localhost;Database=zilean-test;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=30;CommandTimeout=3600;"
       }
     }
   }

src/Zilean.Scraper/Features/Ingestion/Endpoints/GenericIngestionScraping.cs

@@ -61,11 +61,7 @@ private void AddZileanInstancesToUrls(List<GenericEndpoint> urlsToProcess)
         if (configuration.Ingestion.ZileanInstances.Count > 0)
         {
             logger.LogInformation("Adding Zilean instances to the list of URLs to process");
-            urlsToProcess.AddRange(configuration.Ingestion.ZileanInstances.Select(url => new GenericEndpoint
-            {
-                EndpointType = GenericEndpointType.Zilean,
-                Url = url.Url,
-            }));
+            urlsToProcess.AddRange(configuration.Ingestion.ZileanInstances);
         }
     }
 
@@ -74,11 +70,7 @@ private void AddZurgInstancesToUrls(List<GenericEndpoint> urlsToProcess)
         if (configuration.Ingestion.ZurgInstances.Count > 0)
         {
             logger.LogInformation("Adding Zurg instances to the list of URLs to process");
-            urlsToProcess.AddRange(configuration.Ingestion.ZurgInstances.Select(url => new GenericEndpoint
-            {
-                EndpointType = GenericEndpointType.Zurg,
-                Url = url.Url,
-            }));
+            urlsToProcess.AddRange(configuration.Ingestion.ZurgInstances);
         }
     }
 

src/Zilean.Scraper/Features/Ingestion/Processing/StreamedEntryProcessor.cs

@@ -43,6 +43,11 @@ private async Task ProduceEntriesAsync(ChannelWriter<Task<StreamedEntry>> writer
                 _ => throw new InvalidOperationException($"Unknown endpoint type: {_currentEndpoint.EndpointType}")
             };
 
+            if (_currentEndpoint.EndpointType == GenericEndpointType.Zilean)
+            {
+                httpClient.DefaultRequestHeaders.Add("X-Api-Key", _currentEndpoint.ApiKey);
+            }
+
             var response = await httpClient.GetAsync(fullUrl, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
             response.EnsureSuccessStatusCode();
 

src/Zilean.Scraper/Properties/launchSettings.json

@@ -16,7 +16,7 @@
       "commandName": "Project",
       "environmentVariables": {
         "ZILEAN_PYTHON_VENV": "C:\\Python311",
-        "Zilean__Database__ConnectionString": "Host=localhost;Database=zilean-clean2;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=300;CommandTimeout=300;Maximum Pool Size=3;"
+        "Zilean__Database__ConnectionString": "Host=localhost;Database=zilean-test;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=300;CommandTimeout=300;Maximum Pool Size=3;"
       },
       "commandLineArgs": "generic-sync"
     }

src/Zilean.Shared/Features/Configuration/TorrentsConfiguration.cs

@@ -3,4 +3,5 @@
 public class TorrentsConfiguration
 {
     public bool EnableEndpoint { get; set; } = false;
+    public int MaxHashesToCheck { get; set; } = 100;
 }

src/Zilean.Shared/Features/Scraping/GenericEndpoint.cs

@@ -4,4 +4,5 @@ public class GenericEndpoint
 {
     public string? Url { get; set; }
     public GenericEndpointType? EndpointType { get; set; }
+    public string? ApiKey { get; set; }
 }