Skip to content

Commit

Permalink
Refactored code, and added Amiga CDTV and CD32, and ColecoVision to t…
Browse files Browse the repository at this point in the history 
…he PlatformMap (#252)

* More bug fixes

* Update PlatformMap for Amiga CDTV and CD32, and ColecoVision

* Fixed default platform setting for library scan

* Refactor of rematcher

* Temp unzips are no longer deleted immediately - now kept and cleaned up after 5 minutes

* Library Scan now spawns worker processes to perform scans in parallel. Number of workers is limited by MaxWorkers default = 4

* More logging

* More null reference checks

* Overhaul of ROM and MediaGroup handling in web page

* Minor collections updates

* Newlines are now replaced with breaks in HTML on Game summary page
  • Loading branch information
@michael-j-green
michael-j-green authored Jan 10, 2024
1 parent 7d5419d commit 7f2e186
Show file tree
Hide file tree
Showing 33 changed files with 975 additions and 482 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -403,3 +403,4 @@ ASALocalRun/

# Local History for Visual Studio
.localhistory/
gaseous-server/.DS_Store
65 changes: 58 additions & 7 deletions gaseous-server/Classes/Collections.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Mvc.Filters;
using Newtonsoft.Json;
using SharpCompress.Common;

namespace gaseous_server.Classes
{
Expand Down Expand Up @@ -66,7 +67,7 @@ public static CollectionItem GetCollection(long Id) {
public static CollectionItem NewCollection(CollectionItem item)
{
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "INSERT INTO RomCollections (`Name`, Description, Platforms, Genres, Players, PlayerPerspectives, Themes, MinimumRating, MaximumRating, MaximumRomsPerPlatform, MaximumBytesPerPlatform, MaximumCollectionSizeInBytes, FolderStructure, IncludeBIOSFiles, AlwaysInclude, BuiltStatus) VALUES (@name, @description, @platforms, @genres, @players, @playerperspectives, @themes, @minimumrating, @maximumrating, @maximumromsperplatform, @maximumbytesperplatform, @maximumcollectionsizeinbytes, @folderstructure, @includebiosfiles, @alwaysinclude, @builtstatus); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
string sql = "INSERT INTO RomCollections (`Name`, Description, Platforms, Genres, Players, PlayerPerspectives, Themes, MinimumRating, MaximumRating, MaximumRomsPerPlatform, MaximumBytesPerPlatform, MaximumCollectionSizeInBytes, FolderStructure, IncludeBIOSFiles, ArchiveType, AlwaysInclude, BuiltStatus) VALUES (@name, @description, @platforms, @genres, @players, @playerperspectives, @themes, @minimumrating, @maximumrating, @maximumromsperplatform, @maximumbytesperplatform, @maximumcollectionsizeinbytes, @folderstructure, @includebiosfiles, @archivetype, @alwaysinclude, @builtstatus); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
dbDict.Add("name", item.Name);
dbDict.Add("description", item.Description);
Expand All @@ -82,6 +83,7 @@ public static CollectionItem NewCollection(CollectionItem item)
dbDict.Add("maximumcollectionsizeinbytes", Common.ReturnValueIfNull(item.MaximumCollectionSizeInBytes, -1));
dbDict.Add("folderstructure", Common.ReturnValueIfNull(item.FolderStructure, CollectionItem.FolderStructures.Gaseous));
dbDict.Add("includebiosfiles", Common.ReturnValueIfNull(item.IncludeBIOSFiles, 0));
dbDict.Add("archivetype", Common.ReturnValueIfNull(item.ArchiveType, CollectionItem.ArchiveTypes.Zip));
dbDict.Add("alwaysinclude", Newtonsoft.Json.JsonConvert.SerializeObject(Common.ReturnValueIfNull(item.AlwaysInclude, new List<CollectionItem.AlwaysIncludeItem>())));
dbDict.Add("builtstatus", CollectionItem.CollectionBuildStatus.WaitingForBuild);
DataTable romDT = db.ExecuteCMD(sql, dbDict);
Expand All @@ -97,7 +99,7 @@ public static CollectionItem NewCollection(CollectionItem item)
public static CollectionItem EditCollection(long Id, CollectionItem item, bool ForceRebuild = true)
{
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "UPDATE RomCollections SET `Name`=@name, Description=@description, Platforms=@platforms, Genres=@genres, Players=@players, PlayerPerspectives=@playerperspectives, Themes=@themes, MinimumRating=@minimumrating, MaximumRating=@maximumrating, MaximumRomsPerPlatform=@maximumromsperplatform, MaximumBytesPerPlatform=@maximumbytesperplatform, MaximumCollectionSizeInBytes=@maximumcollectionsizeinbytes, FolderStructure=@folderstructure, IncludeBIOSFiles=@includebiosfiles, AlwaysInclude=@alwaysinclude, BuiltStatus=@builtstatus WHERE Id=@id";
string sql = "UPDATE RomCollections SET `Name`=@name, Description=@description, Platforms=@platforms, Genres=@genres, Players=@players, PlayerPerspectives=@playerperspectives, Themes=@themes, MinimumRating=@minimumrating, MaximumRating=@maximumrating, MaximumRomsPerPlatform=@maximumromsperplatform, MaximumBytesPerPlatform=@maximumbytesperplatform, MaximumCollectionSizeInBytes=@maximumcollectionsizeinbytes, FolderStructure=@folderstructure, IncludeBIOSFiles=@includebiosfiles, ArchiveType=@archivetype, AlwaysInclude=@alwaysinclude, BuiltStatus=@builtstatus WHERE Id=@id";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
dbDict.Add("id", Id);
dbDict.Add("name", item.Name);
Expand All @@ -115,8 +117,9 @@ public static CollectionItem EditCollection(long Id, CollectionItem item, bool F
dbDict.Add("folderstructure", Common.ReturnValueIfNull(item.FolderStructure, CollectionItem.FolderStructures.Gaseous));
dbDict.Add("includebiosfiles", Common.ReturnValueIfNull(item.IncludeBIOSFiles, 0));
dbDict.Add("alwaysinclude", Newtonsoft.Json.JsonConvert.SerializeObject(Common.ReturnValueIfNull(item.AlwaysInclude, new List<CollectionItem.AlwaysIncludeItem>())));
dbDict.Add("archivetype", Common.ReturnValueIfNull(item.ArchiveType, CollectionItem.ArchiveTypes.Zip));

string CollectionZipFile = Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, Id + ".zip");
string CollectionZipFile = Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, Id + item.ArchiveExtension);
if (ForceRebuild == true)
{
dbDict.Add("builtstatus", CollectionItem.CollectionBuildStatus.WaitingForBuild);
Expand Down Expand Up @@ -389,7 +392,7 @@ public static void CompileCollections(long CollectionId)
db.ExecuteCMD(sql, dbDict);

List<CollectionContents.CollectionPlatformItem> collectionPlatformItems = GetCollectionContent(collectionItem).Collection;
string ZipFilePath = Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, collectionItem.Id + ".zip");
string ZipFilePath = Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, collectionItem.Id + collectionItem.ArchiveExtension);
string ZipFileTempPath = Path.Combine(Config.LibraryConfiguration.LibraryTempDirectory, collectionItem.Id.ToString());

try
Expand Down Expand Up @@ -508,7 +511,21 @@ public static void CompileCollections(long CollectionId)

// compress to zip
Logging.Log(Logging.LogType.Information, "Collections", "Compressing collection");
ZipFile.CreateFromDirectory(ZipFileTempPath, ZipFilePath, CompressionLevel.SmallestSize, false);
switch(collectionItem.ArchiveType)
{
case CollectionItem.ArchiveTypes.Zip:
ZipFile.CreateFromDirectory(ZipFileTempPath, ZipFilePath, CompressionLevel.SmallestSize, false);
break;

case CollectionItem.ArchiveTypes.RAR:

break;

case CollectionItem.ArchiveTypes.SevenZip:

break;
}


// clean up
if (Directory.Exists(ZipFileTempPath))
Expand Down Expand Up @@ -567,6 +584,7 @@ private static CollectionItem BuildCollectionItem(DataRow row) {
item.MaximumCollectionSizeInBytes = (long)Common.ReturnValueIfNull(row["MaximumCollectionSizeInBytes"], (long)-1);
item.FolderStructure = (CollectionItem.FolderStructures)(int)Common.ReturnValueIfNull(row["FolderStructure"], 0);
item.IncludeBIOSFiles = (bool)row["IncludeBIOSFiles"];
item.ArchiveType = (CollectionItem.ArchiveTypes)(int)Common.ReturnValueIfNull(row["ArchiveType"], 0);
item.AlwaysInclude = Newtonsoft.Json.JsonConvert.DeserializeObject<List<CollectionItem.AlwaysIncludeItem>>(strAlwaysInclude);
item.BuildStatus = (CollectionItem.CollectionBuildStatus)(int)Common.ReturnValueIfNull(row["BuiltStatus"], 0);

Expand Down Expand Up @@ -595,6 +613,32 @@ public CollectionItem()
public long? MaximumCollectionSizeInBytes { get; set; }
public FolderStructures FolderStructure { get; set; } = FolderStructures.Gaseous;
public bool IncludeBIOSFiles { get; set; } = true;
public ArchiveTypes ArchiveType { get; set; } = CollectionItem.ArchiveTypes.Zip;
public string ArchiveExtension
{
get
{
if (ArchiveType != null)
{
switch (ArchiveType)
{
case ArchiveTypes.Zip:
default:
return ".zip";

case ArchiveTypes.RAR:
return ".rar";

case ArchiveTypes.SevenZip:
return ".7z";
}
}
else
{
return ".zip";
}
}
}
public List<AlwaysIncludeItem> AlwaysInclude { get; set; }

[JsonIgnore]
Expand All @@ -604,7 +648,7 @@ public CollectionBuildStatus BuildStatus
{
if (_BuildStatus == CollectionBuildStatus.Completed)
{
if (File.Exists(Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, Id + ".zip")))
if (File.Exists(Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, Id + ArchiveExtension)))
{
return CollectionBuildStatus.Completed;
}
Expand Down Expand Up @@ -632,7 +676,7 @@ public long CollectionBuiltSizeBytes
{
if (BuildStatus == CollectionBuildStatus.Completed)
{
string ZipFilePath = Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, Id + ".zip");
string ZipFilePath = Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, Id + ArchiveExtension);
if (File.Exists(ZipFilePath))
{
FileInfo fi = new FileInfo(ZipFilePath);
Expand Down Expand Up @@ -665,6 +709,13 @@ public enum FolderStructures
RetroPie = 1
}

public enum ArchiveTypes
{
Zip = 0,
RAR = 1,
SevenZip = 2
}

public class AlwaysIncludeItem
{
public long PlatformId { get; set; }
Expand Down
10 changes: 10 additions & 0 deletions gaseous-server/Classes/Config.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,14 @@ private static HasheousClient.Models.MetadataModel.SignatureSources _SignatureSo
}
}

private static int _MaxLibraryScanWorkers
{
get
{
return 4;
}
}

private static string _HasheousHost
{
get
Expand All @@ -522,6 +530,8 @@ private static string _HasheousHost

public HasheousClient.Models.MetadataModel.SignatureSources SignatureSource = _SignatureSource;

public int MaxLibraryScanWorkers = _MaxLibraryScanWorkers;

public string HasheousHost = _HasheousHost;
}

Expand Down
Loading

0 comments on commit 7f2e186

Please sign in to comment.