Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Split LibraryScan and Rematching into separate background tasks #166

Merged
merged 2 commits into from
Oct 17, 2023
Merged

Split LibraryScan and Rematching into separate background tasks #166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
77b04c2
Simplified LibraryScan service and moved rematching to it's own service
michael-j-green Oct 17, 2023
3adee17
Added rematcher to tasks blocked by OrganiseLibrary
michael-j-green Oct 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 53 additions & 32 deletions gaseous-server/Classes/ImportGames.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using gaseous_server.Classes.Metadata;
using gaseous_tools;
using IGDB.Models;
using NuGet.Common;
using static gaseous_server.Classes.Metadata.Games;

namespace gaseous_server.Classes
Expand Down Expand Up @@ -428,7 +429,7 @@ public static long StoreROM(GameLibrary.LibraryItem library, Common.hashObject h
sql = "INSERT INTO Games_Roms (PlatformId, GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Attributes, RomType, RomTypeMedia, MediaLabel, Path, MetadataSource, MetadataGameName, MetadataVersion, LibraryId) VALUES (@platformid, @gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @Attributes, @romtype, @romtypemedia, @medialabel, @path, @metadatasource, @metadatagamename, @metadataversion, @libraryid); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
} else
{
sql = "UPDATE Games_Roms SET PlatformId=platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Attributes=@Attributes, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource, MetadataGameName=@metadatagamename, MetadataVersion=@metadataversion WHERE Id=@id;";
sql = "UPDATE Games_Roms SET PlatformId=@platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Attributes=@Attributes, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource, MetadataGameName=@metadatagamename, MetadataVersion=@metadataversion WHERE Id=@id;";
dbDict.Add("id", UpdateId);
}
dbDict.Add("platformid", Common.ReturnValueIfNull(determinedPlatform.Id, 0));
Expand Down Expand Up @@ -720,37 +721,6 @@ public static void LibraryScan()

if (File.Exists(romPath))
{
// file exists, so lets check to make sure the signature was matched, and update if a signature can be found
if (
romMetadataSource == gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None ||
(int)dtRoms.Rows[i]["MetadataVersion"] == 1
)
{
Common.hashObject hash = new Common.hashObject
{
md5hash = (string)dtRoms.Rows[i]["MD5"],
sha1hash = (string)dtRoms.Rows[i]["SHA1"]
};
FileInfo fi = new FileInfo(romPath);

Models.Signatures_Games sig = GetFileSignature(hash, fi, romPath);
if (sig.Rom.SignatureSource != gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None)
{
Logging.Log(Logging.LogType.Information, "Library Scan", " Update signature found for " + romPath);

// get discovered platform
IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(sig.Flags.IGDBPlatformId);
if (determinedPlatform == null)
{
determinedPlatform = new IGDB.Models.Platform();
}

IGDB.Models.Game determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId);

StoreROM(library, hash, determinedGame, determinedPlatform, sig, romPath, romId);
}
}

if (library.IsDefaultLibrary == true)
{
if (romPath != ComputeROMPath(romId))
Expand All @@ -776,6 +746,57 @@ public static void LibraryScan()
Logging.Log(Logging.LogType.Information, "Library Scan", "Library scan completed");
}
}

public static void Rematcher()
{
// rescan all titles with an unknown platform or title and see if we can get a match
Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch scan starting");

Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "SELECT * FROM Games_Roms WHERE (PlatformId = 0 OR GameId = 0) AND (LastMatchAttemptDate IS NULL OR LastMatchAttemptDate < @lastmatchattemptdate) LIMIT 100;";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
dbDict.Add("lastmatchattemptdate", DateTime.UtcNow.AddMonths(-1));
DataTable data = db.ExecuteCMD(sql, dbDict);
foreach (DataRow row in data.Rows)
{
// get library
GameLibrary.LibraryItem library = GameLibrary.GetLibrary((int)row["LibraryId"]);

// get rom info
long romId = (long)row["Id"];
string romPath = (string)row["Path"];
Common.hashObject hash = new Common.hashObject
{
md5hash = (string)row["MD5"],
sha1hash = (string)row["SHA1"]
};
FileInfo fi = new FileInfo(romPath);

Logging.Log(Logging.LogType.Information, "Rematch Scan", "Running rematch against " + romPath);

// determine rom signature
Models.Signatures_Games sig = GetFileSignature(hash, fi, romPath);

// determine rom platform
IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(sig.Flags.IGDBPlatformId);
if (determinedPlatform == null)
{
determinedPlatform = new IGDB.Models.Platform();
}

IGDB.Models.Game determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId);

StoreROM(library, hash, determinedGame, determinedPlatform, sig, romPath, romId);

string attemptSql = "UPDATE Games_Roms SET LastMatchAttemptDate=@lastmatchattemptdate WHERE Id=@id;";
Dictionary<string, object> dbLastAttemptDict = new Dictionary<string, object>();
dbLastAttemptDict.Add("id", romId);
dbLastAttemptDict.Add("lastmatchattemptdate", DateTime.UtcNow);
db.ExecuteCMD(attemptSql, dbLastAttemptDict);
}

Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch scan completed");
}
}
}

3 changes: 2 additions & 1 deletion gaseous-server/Models/PlatformMapping.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Linq;
using System.Reflection;
using System.Text.Json.Serialization;
using System.Web;
using gaseous_server.Classes;
using gaseous_server.Classes.Metadata;
using gaseous_server.Controllers;
Expand Down Expand Up @@ -201,7 +202,7 @@ public static void WritePlatformMap(PlatformMapItem item, bool Update, bool Allo
sql = "INSERT INTO PlatformMap_AlternateNames (Id, Name) VALUES (@Id, @Name);";
dbDict.Clear();
dbDict.Add("Id", item.IGDBId);
dbDict.Add("Name", alternateName);
dbDict.Add("Name", HttpUtility.HtmlDecode(alternateName));
db.ExecuteCMD(sql, dbDict);
}
}
Expand Down
15 changes: 15 additions & 0 deletions gaseous-server/ProcessQueue.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,14 @@ public void Execute()

break;

case QueueItemType.Rematcher:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Rematch");
Classes.ImportGame.Rematcher();

_SaveLastRunTime = true;

break;

case QueueItemType.CollectionCompiler:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Collection Compiler");
Classes.Collections.CompileCollections((long)Options);
Expand All @@ -168,6 +176,8 @@ public void Execute()
_ForceExecute = false;
_ItemState = QueueItemState.Stopped;
_LastFinishTime = DateTime.UtcNow;

Logging.Log(Logging.LogType.Information, "Timered Event", "Total " + _ItemType + " run time = " + (DateTime.UtcNow - _LastRunTime).TotalSeconds);
}
}
}
Expand Down Expand Up @@ -220,6 +230,11 @@ public enum QueueItemType
/// </summary>
LibraryScan,

/// <summary>
/// Looks for roms in the library that have an unknown platform or game match
/// </summary>
Rematcher,

/// <summary>
/// Builds collections - set the options attribute to the id of the collection to build
/// </summary>
Expand Down
39 changes: 32 additions & 7 deletions gaseous-server/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -182,29 +182,54 @@
PlatformMapping.ExtractPlatformMap();

// add background tasks
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.SignatureIngestor, 60));
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.TitleIngestor, 1,
ProcessQueue.QueueItemType.SignatureIngestor,
60
)
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.TitleIngestor,
1,
new List<ProcessQueue.QueueItemType>
{
ProcessQueue.QueueItemType.OrganiseLibrary,
ProcessQueue.QueueItemType.LibraryScan
})
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.MetadataRefresh, 360));
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.OrganiseLibrary, 1440, new List<ProcessQueue.QueueItemType>
ProcessQueue.QueueItemType.MetadataRefresh,
360
)
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.OrganiseLibrary,
1440,
new List<ProcessQueue.QueueItemType>
{
ProcessQueue.QueueItemType.LibraryScan,
ProcessQueue.QueueItemType.TitleIngestor
ProcessQueue.QueueItemType.TitleIngestor,
ProcessQueue.QueueItemType.Rematcher
})
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.LibraryScan, 1440, new List<ProcessQueue.QueueItemType>
ProcessQueue.QueueItemType.LibraryScan,
60,
new List<ProcessQueue.QueueItemType>
{
ProcessQueue.QueueItemType.OrganiseLibrary
ProcessQueue.QueueItemType.OrganiseLibrary,
ProcessQueue.QueueItemType.Rematcher
})
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.Rematcher,
1440,
new List<ProcessQueue.QueueItemType>
{
ProcessQueue.QueueItemType.OrganiseLibrary,
ProcessQueue.QueueItemType.LibraryScan
})
);


Logging.WriteToDiskOnly = false;

Expand Down
10 changes: 5 additions & 5 deletions gaseous-server/Support/PlatformMap.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -882,7 +882,7 @@
"alternateNames": [
"NES",
"Nintendo Entertainment System",
"Nintendo Famicom &amp;amp; Entertainment System"
"Nintendo Famicom & Entertainment System"
],
"extensions": {
"supportedFileExtensions": [
Expand Down Expand Up @@ -1086,7 +1086,7 @@
"alternateNames": [
"Mega CD",
"Sega CD",
"Sega Mega-CD &amp;amp; Sega CD",
"Sega Mega-CD & Sega CD",
"segacd"
],
"extensions": {
Expand Down Expand Up @@ -1188,7 +1188,7 @@
"igdbName": "Sega Master System/Mark III",
"igdbSlug": "sms",
"alternateNames": [
"Sega Mark III &amp;amp; Master System",
"Sega Mark III & Master System",
"Sega Master System",
"Sega Master System/Mark III",
"sms",
Expand Down Expand Up @@ -1248,7 +1248,7 @@
"genesis-slash-megadrive",
"Sega Genesis",
"Sega Mega Drive",
"Sega Mega Drive &amp; Genesis",
"Sega Mega Drive & Genesis",
"Sega Mega Drive/Genesis"
],
"extensions": {
Expand Down Expand Up @@ -1351,7 +1351,7 @@
"igdbName": "Super Nintendo Entertainment System",
"igdbSlug": "snes",
"alternateNames": [
"Nintendo Super Famicom &amp;amp; Super Entertainment System",
"Nintendo Super Famicom & Super Entertainment System",
"SNES",
"SNES, Super Nintendo",
"Super Nintendo",
Expand Down
5 changes: 4 additions & 1 deletion gaseous-tools/Database/MySQL/gaseous-1004.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,7 @@ CREATE TABLE `Relation_Game_Themes` (
`ThemesId` BIGINT NOT NULL,
PRIMARY KEY (`GameId`, `ThemesId`),
INDEX `idx_PrimaryColumn` (`GameId` ASC) VISIBLE
);
);

ALTER TABLE `Games_Roms`
ADD COLUMN `LastMatchAttemptDate` DATETIME NULL AFTER `LibraryId`;
Loading