Merge pull request #23 from iPromKnight/batched-imports

feat: Rework imports
iPromKnight · Sep 10, 2024 · 18efb4c · 18efb4c
2 parents c83ef2c + fec3e96
commit 18efb4c
Show file tree

Hide file tree

Showing 58 changed files with 1,914 additions and 862 deletions.
diff --git a/.run/Zilean.DmmScraper.run.xml b/.run/Zilean.DmmScraper.run.xml
@@ -1,13 +1,13 @@
 <component name="ProjectRunConfigurationManager">
   <configuration default="false" name="Zilean.DmmScraper" type="DotNetProject" factoryName=".NET Project">
-    <option name="EXE_PATH" value="$PROJECT_DIR$/src/Zilean.DmmScraper/bin/Release/net8.0/dmmscraper" />
+    <option name="EXE_PATH" value="$PROJECT_DIR$/src/Zilean.DmmScraper/bin/Debug/net8.0/dmmscraper.exe" />
     <option name="PROGRAM_PARAMETERS" value="" />
-    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/src/Zilean.DmmScraper/bin/Release/net8.0" />
+    <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/src/Zilean.DmmScraper/bin/Debug/net8.0" />
     <option name="PASS_PARENT_ENVS" value="1" />
     <envs>
       <env name="Zilean__ElasticSearch__Url" value="http://localhost:9200" />
-      <env name="ZILEAN_PYTHON_PYLIB" value="/opt/homebrew/opt/[email protected]/Frameworks/Python.framework/Versions/3.11/lib/libpython3.11.dylib" />
-      <env name="ZILEAN_PYTHON_VENV" value="C:\Users\Prom3theu5\.conda\envs\dev" />
+      <env name="ZILEAN_PYTHON_PYLIB" value="C:\Python311\python311.dll" />
+      <env name="ZILEAN_PYTHON_VENV" value="C:\Python311" />
     </envs>
     <option name="USE_EXTERNAL_CONSOLE" value="0" />
     <option name="USE_MONO" value="0" />

diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -5,33 +5,34 @@
     <PackageVersion Include="Coravel" Version="5.0.4" />
     <PackageVersion Include="CsvHelper" Version="33.0.1" />
     <PackageVersion Include="Dapper" Version="2.1.35" />
-    <PackageVersion Include="FluentAssertions" Version="6.12.0" />
+    <PackageVersion Include="EFCore.BulkExtensions.PostgreSql" Version="8.1.1" />
+    <PackageVersion Include="FluentAssertions" Version="6.12.1" />
     <PackageVersion Include="Microsoft.EntityFrameworkCore" Version="8.0.8" />
     <PackageVersion Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.8">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageVersion>
     <PackageVersion Include="Microsoft.EntityFrameworkCore.Relational" Version="8.0.8" />
-    <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="8.8.0" />
+    <PackageVersion Include="Microsoft.Extensions.Http.Resilience" Version="8.9.1" />
     <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Http" Version="8.0.0" />
     <PackageVersion Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="8.0.4" />
     <PackageVersion Include="NSubstitute" Version="5.1.0" />
     <PackageVersion Include="pythonnet" Version="3.0.3" />
     <PackageVersion Include="Serilog.Sinks.Spectre" Version="0.5.0" />
     <PackageVersion Include="Spectre.Console" Version="0.49.2-preview.0.10" />
-    <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.7.0" />
+    <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.7.3" />
     <PackageVersion Include="SimCube.Aspire" Version="1.0.4" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
-    <PackageVersion Include="Testcontainers" Version="3.9.0" />
-    <PackageVersion Include="Testcontainers.PostgreSql" Version="3.9.0" />
-    <PackageVersion Include="Verify.Xunit" Version="26.2.0" />
+    <PackageVersion Include="Testcontainers" Version="3.10.0" />
+    <PackageVersion Include="Testcontainers.PostgreSql" Version="3.10.0" />
+    <PackageVersion Include="Verify.Xunit" Version="26.4.2" />
     <PackageVersion Include="xunit" Version="2.9.0" />
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageVersion>
-    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.10.0" />
+    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
     <PackageVersion Include="coverlet.collector" Version="6.0.2">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

diff --git a/README.md b/README.md
@@ -11,66 +11,49 @@ The DMM import reruns on missing pages every hour.
 
 ## Configuration
 
-There is no configuration for Zilean.
-For persistence, you can mount a volume on the path `/app/data`.
----
-
-## Compose
-```yaml
-volumes:
-  zilean_data:
-
-services:
-  zilean:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    ports:
-      - "8181:8181"
-    volumes:
-      - zilean_data:/app/data
-````
----
-
-## Endpoints
-
-### Search Endpoint
-
-Endpoint
-```bash
-POST /dmm/search
-```
-
-Request Body
 ```json
 {
-    "queryText": "string"
+  "Zilean": {
+    "Dmm": {
+      "EnableScraping": true,
+      "EnableEndpoint": true,
+      "ScrapeSchedule": "0 * * * *",
+      "MaxFilteredResults": 200,
+      "MinimumScoreMatch": 0.85,
+      "ImportBatched": false
+    },
+    "Database": {
+      "ConnectionString": "Host=localhost;Database=zilean;Username=postgres;Password=postgres;Include Error Detail=true;Timeout=300;CommandTimeout=300;"
+    },
+    "Prowlarr": {
+      "EnableEndpoint": true
+    },
+    "Imdb": {
+      "EnableImportMatching": false,
+      "EnableEndpoint": true,
+      "MinimumScoreMatch": 0.85
+    }
+  }
 }
 ```
 
-### Responses
-Success:
-```json
-[
-  {
-    "filename": "string",
-    "infoHash": "string",
-    "filesize": "long"
-  },
-  {
-    "filename": "string",
-    "infoHash": "string",
-    "filesize": "long"
-  }
-]
-```
+Every option you see can be set as an env variable, the env variable name is the same as the json path with double underscores instead of dots.
+For example, `Zilean__Dmm__EnableScraping` would be the env variable for `Zilean.Dmm.EnableScraping`.
+
+A breakdown of all configuration options:
+
+- `Zilean__Dmm__EnableScraping`: Whether to enable the DMM scraping service.
+- `Zilean__Dmm__EnableEndpoint`: Whether to enable the DMM search endpoint.
+- `Zilean__Dmm__ScrapeSchedule`: The cron schedule for the DMM scraping service.
+- `Zilean__Dmm__MaxFilteredResults`: The maximum number of results to return from the DMM search endpoint.
+- `Zilean__Dmm__MinimumScoreMatch`: The minimum score required for a search result to be returned. Values between 0 and 1. Defaults to 0.85.
+- `Zilean__Dmm__ImportBatched`: Whether to import DMM pages in batches. This is for low end systems. Defaults to false. Will make the initial import take longer. A lot longer.
+- `Zilean__Database__ConnectionString`: The connection string for the database (Postgres).
+- `Zilean__Prowlarr__EnableEndpoint`: Whether to enable the Prowlarr search endpoint. (Unused currently).
+- `Zilean__Imdb__EnableImportMatching`: Whether to enable the IMDB import matching service. Defaults to true. Disabling this will improve import speed at the cost of not having IMDB data.
+- `Zilean__Imdb__EnableEndpoint`: Whether to enable the IMDB search endpoint.
+- `Zilean__Imdb__MinimumScoreMatch`: The minimum score required for a search result to be returned. Values between 0 and 1. Defaults to 0.85.
+---
 
-Error:
-```json
-{
-  "type": "https://tools.ietf.org/html/rfc7231#section-6.5.1",
-  "title": "string",
-  "status": 500,
-  "detail": "string"
-}
-```
+## Compose Example
+See the file [compose.yaml](https://github.com/iPromKnight/zilean/blob/main/compose.yaml) for an example of how to run Zilean.
diff --git a/Zilean.sln.DotSettings.user b/Zilean.sln.DotSettings.user
@@ -1,21 +1,26 @@
 <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
-	<s:String x:Key="/Default/Environment/UnitTesting/UnitTestSessionStore/Sessions/=1182152c_002De1fa_002D4b35_002D8711_002D79eeef4fbfc0/@EntryIndexedValue">&lt;SessionState ContinuousTestingMode="0" Name="VerifyShawshankMatches_WithoutThe_ShouldMatch" xmlns="urn:schemas-jetbrains-com:jetbrains-ut-session"&gt;
-  &lt;TestAncestor&gt;
-    &lt;TestId&gt;xUnit::3F8985A4-7E1C-48A0-89A5-47790470023E::net8.0::Zilean.Tests.Tests.PttPythonTests&lt;/TestId&gt;
-  &lt;/TestAncestor&gt;
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ABulkConfig_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fd6b09cbbbc4e7bf228e82618e4e0859c91be3b93407de754227ae1a771ea2a6_003FBulkConfig_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADbContextBulkExtensions_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fd1f8f68864fcaf754b3aefb6cb6777e56a17cfd9217db17992967674aa5cd1d_003FDbContextBulkExtensions_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ADbContextBulkTransaction_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003F1dfe90a67dda13ce28d4a1975cde7e7445f47ecbb8b4d73dd1bcc0c01acd88_003FDbContextBulkTransaction_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AExceptionDispatchInfo_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fbd1d5c50194fea68ff3559c160230b0ab50f5acf4ce3061bffd6d62958e2182_003FExceptionDispatchInfo_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AJsonSerializer_002Ecs_002Fl_003A_002E_002E_003F_002E_002E_003F_002E_002E_003FUsers_003FProm3theu5_003FAppData_003FRoaming_003FJetBrains_003FRider2024_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F5148e388a7db4994ad2ab3750386454116e910_003F72_003Fc63c4ed4_003FJsonSerializer_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+	<s:String x:Key="/Default/Environment/UnitTesting/UnitTestSessionStore/Sessions/=1182152c_002De1fa_002D4b35_002D8711_002D79eeef4fbfc0/@EntryIndexedValue">&lt;SessionState ContinuousTestingMode="0" Name="VerifyShawshankMatches_WithoutThe_ShouldMatch" xmlns="urn:schemas-jetbrains-com:jetbrains-ut-session"&gt;&#xD;
+  &lt;TestAncestor&gt;&#xD;
+    &lt;TestId&gt;xUnit::3F8985A4-7E1C-48A0-89A5-47790470023E::net8.0::Zilean.Tests.Tests.PttPythonTests&lt;/TestId&gt;&#xD;
+  &lt;/TestAncestor&gt;&#xD;
 &lt;/SessionState&gt;</s:String>
-	<s:String x:Key="/Default/Environment/UnitTesting/UnitTestSessionStore/Sessions/=78cae2e2_002D0b4c_002D410b_002D8d6b_002De16e6ad72a00/@EntryIndexedValue">&lt;SessionState ContinuousTestingMode="0" IsActive="True" Name="IsAMovie_ShouldValidate_ReturnsTrue" xmlns="urn:schemas-jetbrains-com:jetbrains-ut-session"&gt;
-  &lt;TestAncestor&gt;
-    &lt;TestId&gt;xUnit::3F8985A4-7E1C-48A0-89A5-47790470023E::net8.0::Zilean.Tests.Tests.PttPythonTests&lt;/TestId&gt;
-  &lt;/TestAncestor&gt;
+	<s:String x:Key="/Default/Environment/UnitTesting/UnitTestSessionStore/Sessions/=78cae2e2_002D0b4c_002D410b_002D8d6b_002De16e6ad72a00/@EntryIndexedValue">&lt;SessionState ContinuousTestingMode="0" IsActive="True" Name="IsAMovie_ShouldValidate_ReturnsTrue" xmlns="urn:schemas-jetbrains-com:jetbrains-ut-session"&gt;&#xD;
+  &lt;TestAncestor&gt;&#xD;
+    &lt;TestId&gt;xUnit::3F8985A4-7E1C-48A0-89A5-47790470023E::net8.0::Zilean.Tests.Tests.PttPythonTests&lt;/TestId&gt;&#xD;
+  &lt;/TestAncestor&gt;&#xD;
 &lt;/SessionState&gt;</s:String>
-	<s:String x:Key="/Default/Environment/UnitTesting/UnitTestSessionStore/Sessions/=96946a3d_002D7265_002D4ac9_002D82b3_002De9a2bd463785/@EntryIndexedValue">&lt;SessionState ContinuousTestingMode="0" Name="EnsureInitAndStopSuccessfully" xmlns="urn:schemas-jetbrains-com:jetbrains-ut-session"&gt;
-  &lt;TestAncestor&gt;
-    &lt;TestId&gt;xUnit::3F8985A4-7E1C-48A0-89A5-47790470023E::net8.0::Zilean.Tests.Tests.PttPythonTests&lt;/TestId&gt;
-  &lt;/TestAncestor&gt;
+	<s:String x:Key="/Default/Environment/UnitTesting/UnitTestSessionStore/Sessions/=96946a3d_002D7265_002D4ac9_002D82b3_002De9a2bd463785/@EntryIndexedValue">&lt;SessionState ContinuousTestingMode="0" Name="EnsureInitAndStopSuccessfully" xmlns="urn:schemas-jetbrains-com:jetbrains-ut-session"&gt;&#xD;
+  &lt;TestAncestor&gt;&#xD;
+    &lt;TestId&gt;xUnit::3F8985A4-7E1C-48A0-89A5-47790470023E::net8.0::Zilean.Tests.Tests.PttPythonTests&lt;/TestId&gt;&#xD;
+  &lt;/TestAncestor&gt;&#xD;
 &lt;/SessionState&gt;</s:String>
-	<s:String x:Key="/Default/Environment/UnitTesting/UnitTestSessionStore/Sessions/=f4982ea5_002D6a9a_002D4974_002Dac2d_002D9c3d8b90ed5f/@EntryIndexedValue">&lt;SessionState ContinuousTestingMode="0" Name="IsAMovie_ShouldValidate_ReturnsTrue #2" xmlns="urn:schemas-jetbrains-com:jetbrains-ut-session"&gt;
-  &lt;TestAncestor&gt;
-    &lt;TestId&gt;xUnit::3F8985A4-7E1C-48A0-89A5-47790470023E::net8.0::Zilean.Tests.Tests.PttPythonTests&lt;/TestId&gt;
-  &lt;/TestAncestor&gt;
+	<s:String x:Key="/Default/Environment/UnitTesting/UnitTestSessionStore/Sessions/=f4982ea5_002D6a9a_002D4974_002Dac2d_002D9c3d8b90ed5f/@EntryIndexedValue">&lt;SessionState ContinuousTestingMode="0" Name="IsAMovie_ShouldValidate_ReturnsTrue #2" xmlns="urn:schemas-jetbrains-com:jetbrains-ut-session"&gt;&#xD;
+  &lt;TestAncestor&gt;&#xD;
+    &lt;TestId&gt;xUnit::3F8985A4-7E1C-48A0-89A5-47790470023E::net8.0::Zilean.Tests.Tests.PttPythonTests&lt;/TestId&gt;&#xD;
+  &lt;/TestAncestor&gt;&#xD;
 &lt;/SessionState&gt;</s:String></wpf:ResourceDictionary>
diff --git a/compose.yaml b/compose.yaml
@@ -15,9 +15,12 @@ services:
       - zilean_data:/app/data
     environment:
         Zilean__Database__ConnectionString: "Host=postgres;Port=5432;Database=zilean;Username=postgres;Password=postgres"
-      # Zilean__Dmm__ImportBatched: "true"         Allows enabling batched import - this is for low-end systems.
-      # Zilean__Dmm__MaxFilteredResults: 200       Allows changing the maximum number of filtered results returned by the DMM API. 200 is the default.
-      # Zilean__Dmm__MinimumScoreMatch: 0.85       Allows changing the minimum score match for the DMM API. 0.85 is the default. Values between 0 and 1 are accepted.
+      # Zilean__Dmm__ImportBatched: "true"                    Allows enabling batched import - this is for low-end systems.
+      # Zilean__Dmm__MaxFilteredResults: 200                  Allows changing the maximum number of filtered results returned by the DMM API. 200 is the default.
+      # Zilean__Dmm__MinimumScoreMatch: 0.85                  Allows changing the minimum score match for the DMM API. 0.85 is the default. Values between 0 and 1 are accepted.
+      # Zilean__Imdb__MinimumScoreMatch: 0.85                 Allows changing the minimum score match for Imdb Matching API. 0.85 is the default. Values between 0 and 1 are accepted.
+      # Zilean__Dmm__MinimumReDownloadIntervalMinutes: 30     Minimum number of minutes between downloads from the DMM Repo - defaults to `30`
+      # Zilean__Imdb__EnableImportMatching: true              Should IMDB functionality be enabled, true/false, defaults to `true`. Disabling will lead to drastically improved import speed on initial run however you will have no internal imdb matching within zilean, so its up to the upstream project utilising zilean to implement that.
     healthcheck:
       test: curl --connect-timeout 10 --silent --show-error --fail http://localhost:8181/healthchecks/ping
       timeout: 60s

diff --git a/eng/install-python-reqs-dmmscraper.ps1 b/eng/install-python-reqs-dmmscraper.ps1
@@ -0,0 +1,3 @@
+Remove-Item -Path ../src/Zilean.DmmScraper/python -Recurse -Force
+New-Item -Path ../src/Zilean.DmmScraper/python -ItemType Directory
+python -m pip install -r ../requirements.txt -t ../src/Zilean.DmmScraper/python/ --no-user
diff --git a/eng/install-python-reqs-dmmscraper.sh b/eng/install-python-reqs-dmmscraper.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-
 rm -rf ../src/Zilean.DmmScraper/python
 mkdir -p ../src/Zilean.DmmScraper/python
 python3.11 -m pip install -r ../requirements.txt -t ../src/Zilean.DmmScraper/python/
diff --git a/requirements.txt b/requirements.txt
@@ -1,3 +1,3 @@
 loguru==0.7.2
-rich==13.7.1
-rank-torrent-name==0.2.23
+rich==13.8.0
+rank-torrent-name==1.0.4