diff --git a/src/Fluxzy.Core/Archiving/ArchiveMetaInformation.cs b/src/Fluxzy.Core/Archiving/ArchiveMetaInformation.cs index b27dae968..cf6fafd3f 100644 --- a/src/Fluxzy.Core/Archiving/ArchiveMetaInformation.cs +++ b/src/Fluxzy.Core/Archiving/ArchiveMetaInformation.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Net; using System.Reflection; using Fluxzy.Rules.Filters; @@ -30,7 +31,7 @@ public class ArchiveMetaInformation /// /// Archive version /// - public string ArchiveVersion { get; set; } = "0.2.0"; + public string ArchiveVersion { get; set; } = "0.3.0"; /// /// Information about the environment where the archive was created. @@ -42,6 +43,23 @@ public class ArchiveMetaInformation /// public string FluxzyVersion { get; set; } = Assembly.GetExecutingAssembly().GetName().Version!.ToString(); + /// + /// Snapshot of the that was active when the archive was produced. + /// Sensitive fields (PKCS#12 password and file path, proxy authentication password, + /// certificate cache directory, user agent configuration file) are scrubbed before serialization. + /// When is true, alteration rules + /// and the save filter are also omitted. + /// Null on archives produced by Fluxzy <= 0.2.0 and on archives produced by import engines. + /// + public FluxzySetting? CapturedSetting { get; set; } + + /// + /// Endpoints the proxy was actually listening on once Proxy.Run() resolved them. + /// Differs from when a configured port of 0 + /// was replaced by an OS-assigned ephemeral port. + /// + public List ResolvedEndPoints { get; set; } = new(); + /// /// Can be used to store additional information about the archive. /// diff --git a/src/Fluxzy.Core/Archiving/GlobalArchiveOption.cs b/src/Fluxzy.Core/Archiving/GlobalArchiveOption.cs index 5db0ed956..e23e2ac5b 100644 --- a/src/Fluxzy.Core/Archiving/GlobalArchiveOption.cs +++ b/src/Fluxzy.Core/Archiving/GlobalArchiveOption.cs @@ -12,8 +12,8 @@ namespace Fluxzy { - /// - /// Provide serialization settings for producing default archive format + /// + /// Provide serialization settings for producing default archive format /// public static class GlobalArchiveOption { @@ -24,8 +24,8 @@ public static class GlobalArchiveOption CompositeResolver.Create(new IMessagePackFormatter[] { new MessagePackAddressFormatter() }, new IFormatterResolver[] { StandardResolverAllowPrivate.Instance, ContractlessStandardResolver.Instance })); - /// - /// STJ default archive option + /// + /// STJ default archive option /// public static JsonSerializerOptions DefaultSerializerOptions { get; } = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -37,13 +37,14 @@ public static class GlobalArchiveOption new IpAddressConverter(), new IpEndPointConverter(), new PolymorphicConverter(), - new PolymorphicConverter() + new PolymorphicConverter(), + new RedactingFluxzySettingConverter() }, NumberHandling = JsonNumberHandling.AllowReadingFromString }; - /// - /// STJ default archive option for configuration file + /// + /// STJ default archive option for configuration file /// public static JsonSerializerOptions ConfigSerializerOptions { get; } = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -61,8 +62,8 @@ public static class GlobalArchiveOption NumberHandling = JsonNumberHandling.AllowReadingFromString }; - /// - /// HAR STJ archiving option, used by Har Packager + /// + /// HAR STJ archiving option, used by Har Packager /// public static JsonSerializerOptions HttpArchiveSerializerOptions { get; } = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -76,8 +77,8 @@ public static class GlobalArchiveOption }; - /// - /// HAR STJ archiving option, used by Har Packager + /// + /// HAR STJ archiving option, used by Har Packager /// public static JsonSerializerOptions HttpArchivePrettySerializerOptions { get; } = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, diff --git a/src/Fluxzy.Core/Archiving/Readers/ArchiveMetaInformationReader.cs b/src/Fluxzy.Core/Archiving/Readers/ArchiveMetaInformationReader.cs new file mode 100644 index 000000000..b6d9fe24e --- /dev/null +++ b/src/Fluxzy.Core/Archiving/Readers/ArchiveMetaInformationReader.cs @@ -0,0 +1,36 @@ +// Copyright 2021 - Haga Rakotoharivelo - https://github.com/haga-rak + +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace Fluxzy.Readers +{ + internal static class ArchiveMetaInformationReader + { + /// + /// Deserializes a meta.json blob, falling back to a meta with a null + /// when the embedded + /// setting can't be deserialized (e.g. a Filter/Action discriminator from a + /// newer plugin). + /// + public static ArchiveMetaInformation Read(byte[] bytes) + { + try { + return JsonSerializer.Deserialize(bytes, + GlobalArchiveOption.DefaultSerializerOptions) ?? new ArchiveMetaInformation(); + } + catch (JsonException) { + var node = JsonNode.Parse(bytes); + + if (node is JsonObject obj && obj.ContainsKey("capturedSetting")) { + obj.Remove("capturedSetting"); + + return JsonSerializer.Deserialize(obj.ToJsonString(), + GlobalArchiveOption.DefaultSerializerOptions) ?? new ArchiveMetaInformation(); + } + + throw; + } + } + } +} diff --git a/src/Fluxzy.Core/Archiving/Readers/DirectoryArchiveReader.cs b/src/Fluxzy.Core/Archiving/Readers/DirectoryArchiveReader.cs index e28f4a716..353c8167d 100644 --- a/src/Fluxzy.Core/Archiving/Readers/DirectoryArchiveReader.cs +++ b/src/Fluxzy.Core/Archiving/Readers/DirectoryArchiveReader.cs @@ -47,10 +47,8 @@ public ArchiveMetaInformation ReadMetaInformation() return new ArchiveMetaInformation(); } - using var metaStream = File.Open(metaPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - - return JsonSerializer.Deserialize(metaStream, - GlobalArchiveOption.DefaultSerializerOptions)!; + var bytes = File.ReadAllBytes(metaPath); + return ArchiveMetaInformationReader.Read(bytes); } public IEnumerable ReadAllExchanges() diff --git a/src/Fluxzy.Core/Archiving/Readers/FluxzyArchiveReader.cs b/src/Fluxzy.Core/Archiving/Readers/FluxzyArchiveReader.cs index 6893c3f6d..51d9a754d 100644 --- a/src/Fluxzy.Core/Archiving/Readers/FluxzyArchiveReader.cs +++ b/src/Fluxzy.Core/Archiving/Readers/FluxzyArchiveReader.cs @@ -43,9 +43,10 @@ public ArchiveMetaInformation ReadMetaInformation() } using var metaStream = metaEntry.Open(); + using var ms = new MemoryStream(); + metaStream.CopyTo(ms); - return JsonSerializer.Deserialize(metaStream, - GlobalArchiveOption.DefaultSerializerOptions)!; + return ArchiveMetaInformationReader.Read(ms.ToArray()); } public IEnumerable ReadAllExchanges() diff --git a/src/Fluxzy.Core/Archiving/Writers/DirectoryArchiveWriter.cs b/src/Fluxzy.Core/Archiving/Writers/DirectoryArchiveWriter.cs index 9640693a9..476a9b830 100644 --- a/src/Fluxzy.Core/Archiving/Writers/DirectoryArchiveWriter.cs +++ b/src/Fluxzy.Core/Archiving/Writers/DirectoryArchiveWriter.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Net; using System.Runtime.InteropServices; using System.Text.Json; using System.Threading; @@ -16,10 +17,10 @@ namespace Fluxzy.Writers { public class DirectoryArchiveWriter : RealtimeArchiveWriter { - private readonly ArchiveMetaInformation _archiveMetaInformation = CreateNewCaptureArchiveMetaInformation(); + private readonly ArchiveMetaInformation _archiveMetaInformation; private readonly object _metaLock = new object(); - private static ArchiveMetaInformation CreateNewCaptureArchiveMetaInformation() + private static ArchiveMetaInformation CreateNewCaptureArchiveMetaInformation(FluxzySetting? capturedSetting) { var metaInformation = new ArchiveMetaInformation { @@ -31,12 +32,23 @@ private static ArchiveMetaInformation CreateNewCaptureArchiveMetaInformation() "unknown", #endif FluxzySharedSetting.SkipCollectingEnvironmentInformation ? "" : Environment.MachineName - ) + ), + CapturedSetting = FreezeCapturedSetting(capturedSetting) }; return metaInformation; } + private static FluxzySetting? FreezeCapturedSetting(FluxzySetting? capturedSetting) + { + if (capturedSetting == null) + return null; + + // Snapshot the live setting so subsequent mutations don't leak into persisted meta. + var json = JsonSerializer.Serialize(capturedSetting, GlobalArchiveOption.ConfigSerializerOptions); + return JsonSerializer.Deserialize(json, GlobalArchiveOption.ConfigSerializerOptions); + } + private readonly string _archiveMetaInformationPath; private readonly string _baseDirectory; private readonly string _captureDirectory; @@ -47,7 +59,13 @@ private static ArchiveMetaInformation CreateNewCaptureArchiveMetaInformation() private readonly string _connectionDirectory; public DirectoryArchiveWriter(string baseDirectory, Filter? saveFilter) + : this(baseDirectory, saveFilter, capturedSetting: null) + { + } + + public DirectoryArchiveWriter(string baseDirectory, Filter? saveFilter, FluxzySetting? capturedSetting) { + _archiveMetaInformation = CreateNewCaptureArchiveMetaInformation(capturedSetting); _baseDirectory = baseDirectory; _saveFilter = saveFilter; _contentDirectory = Path.Combine(baseDirectory, "contents"); @@ -59,6 +77,14 @@ public DirectoryArchiveWriter(string baseDirectory, Filter? saveFilter) _archiveMetaInformationPath = DirectoryArchiveHelper.GetMetaPath(baseDirectory); } + public void SetResolvedEndPoints(IEnumerable endPoints) + { + lock (_metaLock) { + _archiveMetaInformation.ResolvedEndPoints = endPoints.ToList(); + UpdateMeta(true); + } + } + public override void Init() { base.Init(); diff --git a/src/Fluxzy.Core/FluxzySharedSetting.cs b/src/Fluxzy.Core/FluxzySharedSetting.cs index 0e3b298b6..d94488d9f 100644 --- a/src/Fluxzy.Core/FluxzySharedSetting.cs +++ b/src/Fluxzy.Core/FluxzySharedSetting.cs @@ -22,6 +22,9 @@ static FluxzySharedSetting() SkipCollectingEnvironmentInformation = Environment.GetEnvironmentVariable("SkipCollectingEnvironmentInformation") == "1"; + RedactSettingsInArchive = + Environment.GetEnvironmentVariable("FLUXZY_REDACT_SETTINGS_IN_ARCHIVE") == "1"; + if (string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("appdata"))) { // for macOS and linux, this environment variable used in several temp file (certcache) is not // set leading unwanted folder creation @@ -52,6 +55,14 @@ static FluxzySharedSetting() /// public static bool SkipCollectingEnvironmentInformation { get; set; } + /// + /// When set to true, the FluxzySetting snapshot embedded in archive meta information will + /// omit alteration rules and the save filter (in addition to the always-on redaction of + /// credentials and local file paths). Settable via the FLUXZY_REDACT_SETTINGS_IN_ARCHIVE + /// environment variable. + /// + public static bool RedactSettingsInArchive { get; set; } + /// /// public static int DownStreamProviderReceiveTimeoutMilliseconds { get; } = diff --git a/src/Fluxzy.Core/Misc/Converters/RedactingFluxzySettingConverter.cs b/src/Fluxzy.Core/Misc/Converters/RedactingFluxzySettingConverter.cs new file mode 100644 index 000000000..0fd3999ab --- /dev/null +++ b/src/Fluxzy.Core/Misc/Converters/RedactingFluxzySettingConverter.cs @@ -0,0 +1,55 @@ +// Copyright 2021 - Haga Rakotoharivelo - https://github.com/haga-rak + +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; + +namespace Fluxzy.Misc.Converters +{ + /// + /// Serializes for embedding in archive meta information, + /// scrubbing credentials and local file paths. Reading is delegated to the default + /// contract so consumers see a regular . + /// + internal class RedactingFluxzySettingConverter : JsonConverter + { + public override FluxzySetting? Read(ref Utf8JsonReader reader, System.Type typeToConvert, JsonSerializerOptions options) + { + return JsonSerializer.Deserialize(ref reader, GlobalArchiveOption.ConfigSerializerOptions); + } + + public override void Write(Utf8JsonWriter writer, FluxzySetting value, JsonSerializerOptions options) + { + var node = JsonSerializer.SerializeToNode(value, GlobalArchiveOption.ConfigSerializerOptions)?.AsObject(); + + if (node == null) { + writer.WriteNullValue(); + return; + } + + Redact(node); + + node.WriteTo(writer); + } + + private static void Redact(JsonObject root) + { + if (root["caCertificate"] is JsonObject cert) { + cert.Remove("pkcs12File"); + cert.Remove("pkcs12Password"); + } + + if (root["proxyAuthentication"] is JsonObject auth) { + auth.Remove("password"); + } + + root.Remove("certificateCacheDirectory"); + root.Remove("userAgentActionConfigurationFile"); + + if (FluxzySharedSetting.RedactSettingsInArchive) { + root.Remove("internalAlterationRules"); + root.Remove("saveFilter"); + } + } + } +} diff --git a/src/Fluxzy.Core/Proxy.cs b/src/Fluxzy.Core/Proxy.cs index 6852dc345..92dc5a4dd 100644 --- a/src/Fluxzy.Core/Proxy.cs +++ b/src/Fluxzy.Core/Proxy.cs @@ -115,7 +115,7 @@ public Proxy( Directory.CreateDirectory(StartupSetting.ArchivingPolicy.Directory); Writer = new DirectoryArchiveWriter(StartupSetting.ArchivingPolicy.Directory, - StartupSetting.SaveFilter); + StartupSetting.SaveFilter, StartupSetting); } if (StartupSetting.ArchivingPolicy.Type == ArchivingPolicyType.None) { @@ -348,6 +348,10 @@ public IReadOnlyCollection Run() EndPoints = endPoints; + if (Writer is DirectoryArchiveWriter directoryWriter) { + directoryWriter.SetResolvedEndPoints(endPoints); + } + if (StartupSetting.EnableDiscoveryService) { StartDiscoveryServices(endPoints); } diff --git a/test/Fluxzy.Tests/UnitTests/Archiving/CapturedSettingTests.cs b/test/Fluxzy.Tests/UnitTests/Archiving/CapturedSettingTests.cs new file mode 100644 index 000000000..ae720b485 --- /dev/null +++ b/test/Fluxzy.Tests/UnitTests/Archiving/CapturedSettingTests.cs @@ -0,0 +1,180 @@ +// Copyright 2021 - Haga Rakotoharivelo - https://github.com/haga-rak + +using System; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using Fluxzy.Certificates; +using Fluxzy.Readers; +using Fluxzy.Rules.Actions; +using Fluxzy.Rules.Filters.RequestFilters; +using Fluxzy.Writers; +using Xunit; + +namespace Fluxzy.Tests.UnitTests.Archiving +{ + public class CapturedSettingTests : IDisposable + { + private readonly string _baseDirectory; + + public CapturedSettingTests() + { + _baseDirectory = Path.Combine(Path.GetTempPath(), + "fluxzy-capturedsetting-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(_baseDirectory); + } + + public void Dispose() + { + try { + if (Directory.Exists(_baseDirectory)) + Directory.Delete(_baseDirectory, recursive: true); + } + catch { + // best effort cleanup + } + } + + private static FluxzySetting BuildRichSetting() + { + var setting = FluxzySetting.CreateDefault(); + setting.SetConnectionPerHost(7); + setting.SetExpectContinueTimeout(TimeSpan.FromSeconds(3)); + setting.SetSaveFilter(new HostFilter("example.com")); + setting.AddAlterationRules(new AddRequestHeaderAction("X-Test", "value"), + new HostFilter("example.com")); + return setting; + } + + [Fact] + public void Captured_Setting_Round_Trip_Directory() + { + var setting = BuildRichSetting(); + + var writer = new DirectoryArchiveWriter(_baseDirectory, saveFilter: null, capturedSetting: setting); + writer.Init(); + writer.SetResolvedEndPoints(new[] { new IPEndPoint(IPAddress.Loopback, 12345) }); + writer.Dispose(); + + var reader = new DirectoryArchiveReader(_baseDirectory); + var meta = reader.ReadMetaInformation(); + + Assert.NotNull(meta.CapturedSetting); + Assert.Equal(7, meta.CapturedSetting!.ConnectionPerHost); + Assert.Equal(TimeSpan.FromSeconds(3), meta.CapturedSetting.ExpectContinueTimeout); + Assert.NotNull(meta.CapturedSetting.SaveFilter); + Assert.Single(meta.CapturedSetting.AlterationRules); + + Assert.Single(meta.ResolvedEndPoints); + Assert.Equal(12345, meta.ResolvedEndPoints[0].Port); + + Assert.Equal("0.3.0", meta.ArchiveVersion); + } + + [Fact] + public void Captured_Setting_Redacts_Secrets() + { + var setting = FluxzySetting.CreateDefault(); + setting.SetCaCertificate(Certificate.LoadFromPkcs12("/tmp/secret.p12", "super-secret-passphrase")); + setting.SetProxyAuthentication(ProxyAuthentication.Basic("alice", "hunter2")); + setting.SetCertificateCacheDirectory("/home/alice/.fluxzy/cache"); + + var writer = new DirectoryArchiveWriter(_baseDirectory, saveFilter: null, capturedSetting: setting); + writer.Init(); + writer.Dispose(); + + var metaPath = Path.Combine(_baseDirectory, "meta.json"); + var raw = File.ReadAllText(metaPath); + + Assert.DoesNotContain("super-secret-passphrase", raw); + Assert.DoesNotContain("hunter2", raw); + Assert.DoesNotContain("/tmp/secret.p12", raw); + Assert.DoesNotContain("/home/alice/.fluxzy/cache", raw); + + var reader = new DirectoryArchiveReader(_baseDirectory); + var meta = reader.ReadMetaInformation(); + + Assert.NotNull(meta.CapturedSetting); + Assert.Null(meta.CapturedSetting!.CaCertificate.Pkcs12File); + Assert.Null(meta.CapturedSetting.CaCertificate.Pkcs12Password); + Assert.NotNull(meta.CapturedSetting.ProxyAuthentication); + Assert.Equal("alice", meta.CapturedSetting.ProxyAuthentication!.Username); + Assert.Null(meta.CapturedSetting.ProxyAuthentication.Password); + } + + [Fact] + public async Task Captured_Setting_Round_Trip_Fxzy_Zip() + { + var setting = BuildRichSetting(); + + var writer = new DirectoryArchiveWriter(_baseDirectory, saveFilter: null, capturedSetting: setting); + writer.Init(); + writer.Dispose(); + + var fxzyPath = Path.Combine(_baseDirectory, "..", Guid.NewGuid().ToString("N") + ".fxzy"); + var packager = new FxzyDirectoryPackager(); + await packager.Pack(_baseDirectory, fxzyPath); + + try { + using var zipReader = new FluxzyArchiveReader(fxzyPath); + var meta = zipReader.ReadMetaInformation(); + + Assert.NotNull(meta.CapturedSetting); + Assert.Equal(7, meta.CapturedSetting!.ConnectionPerHost); + Assert.Single(meta.CapturedSetting.AlterationRules); + } + finally { + if (File.Exists(fxzyPath)) + File.Delete(fxzyPath); + } + } + + [Fact] + public void Captured_Setting_Null_When_Absent_Legacy_Archive() + { + using var reader = new FluxzyArchiveReader("_Files/Archives/with-request-payload.fxzy"); + var meta = reader.ReadMetaInformation(); + + Assert.Null(meta.CapturedSetting); + } + + [Fact] + public void RedactSettingsInArchive_Drops_Rules_And_SaveFilter() + { + var setting = BuildRichSetting(); + + var previous = FluxzySharedSetting.RedactSettingsInArchive; + FluxzySharedSetting.RedactSettingsInArchive = true; + + try { + var writer = new DirectoryArchiveWriter(_baseDirectory, saveFilter: null, capturedSetting: setting); + writer.Init(); + writer.Dispose(); + + var reader = new DirectoryArchiveReader(_baseDirectory); + var meta = reader.ReadMetaInformation(); + + Assert.NotNull(meta.CapturedSetting); + Assert.Empty(meta.CapturedSetting!.AlterationRules); + Assert.Null(meta.CapturedSetting.SaveFilter); + } + finally { + FluxzySharedSetting.RedactSettingsInArchive = previous; + } + } + + [Fact] + public void Captured_Setting_Null_When_Not_Provided() + { + var writer = new DirectoryArchiveWriter(_baseDirectory, saveFilter: null); + writer.Init(); + writer.Dispose(); + + var reader = new DirectoryArchiveReader(_baseDirectory); + var meta = reader.ReadMetaInformation(); + + Assert.Null(meta.CapturedSetting); + } + } +}