Skip to content

Commit

Permalink
Add Filesystem Compression as a toggle and button. Also some auto-for…
Browse files Browse the repository at this point in the history 
…matting.
  • Loading branch information
@Ottermandias
Ottermandias committed Sep 14, 2023
1 parent e268739 commit 4e70477
Show file tree
Hide file tree
Showing 21 changed files with 385 additions and 344 deletions.
2 changes: 1 addition & 1 deletion OtterGui
Submodule OtterGui updated 3 files
+12 −0 Compression/CompressionAlgorithm.cs
+193 −0 Compression/FileCompactor.cs
+88 −0 Compression/Interop.cs
1 change: 1 addition & 0 deletions Penumbra/Configuration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ public class Configuration : IPluginConfiguration, ISavable
public bool PrintSuccessfulCommandsToChat { get; set; } = true;
public bool FixMainWindow { get; set; } = false;
public bool AutoDeduplicateOnImport { get; set; } = true;
public bool UseFileSystemCompression { get; set; } = true;
public bool EnableHttpApi { get; set; } = true;

public string DefaultModImportPath { get; set; } = string.Empty;
Expand Down
146 changes: 66 additions & 80 deletions Penumbra/Import/TexToolsImport.cs
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using Penumbra.Api;
using OtterGui.Compression;
using Penumbra.Import.Structs;
using Penumbra.Mods;
using Penumbra.Mods.Editor;
using Penumbra.Mods.Manager;
using FileMode = System.IO.FileMode;
using ZipArchive = SharpCompress.Archives.Zip.ZipArchive;
Expand All @@ -25,41 +25,41 @@ public partial class TexToolsImporter : IDisposable
private readonly DirectoryInfo _baseDirectory;
private readonly string _tmpFile;

private readonly IEnumerable< FileInfo > _modPackFiles;
private readonly IEnumerable<FileInfo> _modPackFiles;
private readonly int _modPackCount;
private FileStream? _tmpFileStream;
private StreamDisposer? _streamDisposer;
private readonly CancellationTokenSource _cancellation = new();
private readonly CancellationToken _token;

public ImporterState State { get; private set; }
public readonly List< (FileInfo File, DirectoryInfo? Mod, Exception? Error) > ExtractedMods;
public ImporterState State { get; private set; }
public readonly List<(FileInfo File, DirectoryInfo? Mod, Exception? Error)> ExtractedMods;

private readonly Configuration _config;
private readonly ModEditor _editor;
private readonly ModManager _modManager;

public TexToolsImporter( int count, IEnumerable< FileInfo > modPackFiles,
Action< FileInfo, DirectoryInfo?, Exception? > handler, Configuration config, ModEditor editor, ModManager modManager)
private readonly ModManager _modManager;
private readonly FileCompactor _compactor;

public TexToolsImporter(int count, IEnumerable<FileInfo> modPackFiles, Action<FileInfo, DirectoryInfo?, Exception?> handler,
Configuration config, ModEditor editor, ModManager modManager, FileCompactor compactor)
{
_baseDirectory = modManager.BasePath;
_tmpFile = Path.Combine( _baseDirectory.FullName, TempFileName );
_tmpFile = Path.Combine(_baseDirectory.FullName, TempFileName);
_modPackFiles = modPackFiles;
_config = config;
_editor = editor;
_modManager = modManager;
_modManager = modManager;
_compactor = compactor;
_modPackCount = count;
ExtractedMods = new List< (FileInfo, DirectoryInfo?, Exception?) >( count );
ExtractedMods = new List<(FileInfo, DirectoryInfo?, Exception?)>(count);
_token = _cancellation.Token;
Task.Run( ImportFiles, _token )
.ContinueWith( _ => CloseStreams() )
.ContinueWith( _ =>
Task.Run(ImportFiles, _token)
.ContinueWith(_ => CloseStreams())
.ContinueWith(_ =>
{
foreach( var (file, dir, error) in ExtractedMods )
{
handler( file, dir, error );
}
} );
foreach (var (file, dir, error) in ExtractedMods)
handler(file, dir, error);
});
}

private void CloseStreams()
Expand All @@ -71,45 +71,43 @@ private void CloseStreams()

public void Dispose()
{
_cancellation.Cancel( true );
if( State != ImporterState.WritingPackToDisk )
_cancellation.Cancel(true);
if (State != ImporterState.WritingPackToDisk)
{
_tmpFileStream?.Dispose();
_tmpFileStream = null;
}

if( State != ImporterState.ExtractingModFiles )
{
if (State != ImporterState.ExtractingModFiles)
ResetStreamDisposer();
}
}

private void ImportFiles()
{
State = ImporterState.None;
_currentModPackIdx = 0;
foreach( var file in _modPackFiles )
State = ImporterState.None;
_currentModPackIdx = 0;
foreach (var file in _modPackFiles)
{
_currentModDirectory = null;
if( _token.IsCancellationRequested )
if (_token.IsCancellationRequested)
{
ExtractedMods.Add( ( file, null, new TaskCanceledException( "Task canceled by user." ) ) );
ExtractedMods.Add((file, null, new TaskCanceledException("Task canceled by user.")));
continue;
}

try
{
var directory = VerifyVersionAndImport( file );
ExtractedMods.Add( ( file, directory, null ) );
if( _config.AutoDeduplicateOnImport )
var directory = VerifyVersionAndImport(file);
ExtractedMods.Add((file, directory, null));
if (_config.AutoDeduplicateOnImport)
{
State = ImporterState.DeduplicatingFiles;
_editor.Duplicates.DeduplicateMod( directory );
_editor.Duplicates.DeduplicateMod(directory);
}
}
catch( Exception e )
catch (Exception e)
{
ExtractedMods.Add( ( file, _currentModDirectory, e ) );
ExtractedMods.Add((file, _currentModDirectory, e));
_currentNumOptions = 0;
_currentOptionIdx = 0;
_currentFileIdx = 0;
Expand All @@ -124,92 +122,80 @@ private void ImportFiles()

// Rudimentary analysis of a TTMP file by extension and version.
// Puts out warnings if extension does not correspond to data.
private DirectoryInfo VerifyVersionAndImport( FileInfo modPackFile )
private DirectoryInfo VerifyVersionAndImport(FileInfo modPackFile)
{
if( modPackFile.Extension.ToLowerInvariant() is ".pmp" or ".zip" or ".7z" or ".rar" )
{
return HandleRegularArchive( modPackFile );
}
if (modPackFile.Extension.ToLowerInvariant() is ".pmp" or ".zip" or ".7z" or ".rar")
return HandleRegularArchive(modPackFile);

using var zfs = modPackFile.OpenRead();
using var extractedModPack = ZipArchive.Open( zfs );
using var extractedModPack = ZipArchive.Open(zfs);

var mpl = FindZipEntry( extractedModPack, "TTMPL.mpl" );
if( mpl == null )
{
throw new FileNotFoundException( "ZIP does not contain a TTMPL.mpl file." );
}
var mpl = FindZipEntry(extractedModPack, "TTMPL.mpl");
if (mpl == null)
throw new FileNotFoundException("ZIP does not contain a TTMPL.mpl file.");

var modRaw = GetStringFromZipEntry( mpl, Encoding.UTF8 );
var modRaw = GetStringFromZipEntry(mpl, Encoding.UTF8);

// At least a better validation than going by the extension.
if( modRaw.Contains( "\"TTMPVersion\":" ) )
if (modRaw.Contains("\"TTMPVersion\":"))
{
if( modPackFile.Extension != ".ttmp2" )
{
Penumbra.Log.Warning( $"File {modPackFile.FullName} seems to be a V2 TTMP, but has the wrong extension." );
}
if (modPackFile.Extension != ".ttmp2")
Penumbra.Log.Warning($"File {modPackFile.FullName} seems to be a V2 TTMP, but has the wrong extension.");

return ImportV2ModPack( modPackFile, extractedModPack, modRaw );
return ImportV2ModPack(modPackFile, extractedModPack, modRaw);
}

if( modPackFile.Extension != ".ttmp" )
{
Penumbra.Log.Warning( $"File {modPackFile.FullName} seems to be a V1 TTMP, but has the wrong extension." );
}
if (modPackFile.Extension != ".ttmp")
Penumbra.Log.Warning($"File {modPackFile.FullName} seems to be a V1 TTMP, but has the wrong extension.");

return ImportV1ModPack( modPackFile, extractedModPack, modRaw );
return ImportV1ModPack(modPackFile, extractedModPack, modRaw);
}

// You can in no way rely on any file paths in TTMPs so we need to just do this, sorry
private static ZipArchiveEntry? FindZipEntry( ZipArchive file, string fileName )
=> file.Entries.FirstOrDefault( e => !e.IsDirectory && e.Key.Contains( fileName ) );
private static ZipArchiveEntry? FindZipEntry(ZipArchive file, string fileName)
=> file.Entries.FirstOrDefault(e => !e.IsDirectory && e.Key.Contains(fileName));

private static string GetStringFromZipEntry( ZipArchiveEntry entry, Encoding encoding )
private static string GetStringFromZipEntry(ZipArchiveEntry entry, Encoding encoding)
{
using var ms = new MemoryStream();
using var s = entry.OpenEntryStream();
s.CopyTo( ms );
return encoding.GetString( ms.ToArray() );
s.CopyTo(ms);
return encoding.GetString(ms.ToArray());
}

private void WriteZipEntryToTempFile( Stream s )
private void WriteZipEntryToTempFile(Stream s)
{
_tmpFileStream?.Dispose(); // should not happen
_tmpFileStream = new FileStream( _tmpFile, FileMode.Create );
if( _token.IsCancellationRequested )
{
_tmpFileStream = new FileStream(_tmpFile, FileMode.Create);
if (_token.IsCancellationRequested)
return;
}

s.CopyTo( _tmpFileStream );
s.CopyTo(_tmpFileStream);
_tmpFileStream.Dispose();
_tmpFileStream = null;
}

private StreamDisposer GetSqPackStreamStream( ZipArchive file, string entryName )
private StreamDisposer GetSqPackStreamStream(ZipArchive file, string entryName)
{
State = ImporterState.WritingPackToDisk;

// write shitty zip garbage to disk
var entry = FindZipEntry( file, entryName );
if( entry == null )
{
throw new FileNotFoundException( $"ZIP does not contain a file named {entryName}." );
}
var entry = FindZipEntry(file, entryName);
if (entry == null)
throw new FileNotFoundException($"ZIP does not contain a file named {entryName}.");

using var s = entry.OpenEntryStream();

WriteZipEntryToTempFile( s );
WriteZipEntryToTempFile(s);

_streamDisposer?.Dispose(); // Should not happen.
var fs = new FileStream( _tmpFile, FileMode.Open );
return new StreamDisposer( fs );
var fs = new FileStream(_tmpFile, FileMode.Open);
return new StreamDisposer(fs);
}

private void ResetStreamDisposer()
{
_streamDisposer?.Dispose();
_streamDisposer = null;
}
}
}
1 change: 0 additions & 1 deletion Penumbra/Import/TexToolsImporter.Gui.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ public partial class TexToolsImporter
private string _currentOptionName = string.Empty;
private string _currentFileName = string.Empty;


public void DrawProgressInfo( Vector2 size )
{
if( _modPackCount == 0 )
Expand Down
Loading

0 comments on commit 4e70477

Please sign in to comment.