Skip to content

Commit

Permalink
fixed logging and improved background task execution
Browse files Browse the repository at this point in the history
  • Loading branch information
@mplogas
mplogas committed Jun 30, 2024
1 parent 83c6b73 commit 59ede69
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 103 deletions.
29 changes: 10 additions & 19 deletions KernelMemory.FileWatcher/KernelMemory.FileWatcher.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,35 +5,26 @@
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
<DockerfileRunArguments>-v G:\docker\km-filewatcher\appsettings.json:/config/appsettings.json:rw -v G:\tmp\km-filewatcher\folder_02:/data/folder_02:ro</DockerfileRunArguments>
<ContainerRepository>km-filewatcher</ContainerRepository>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0">
<TreatAsUsed>true</TreatAsUsed>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0">
<TreatAsUsed>true</TreatAsUsed>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.3" />
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0">
<TreatAsUsed>true</TreatAsUsed>
</PackageReference>
<PackageReference Include="Microsoft.Extensions.Logging" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.ObjectPool" Version="8.0.3" />
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.5" />
<PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
<PackageReference Include="Serilog" Version="3.1.1">
<TreatAsUsed>true</TreatAsUsed>
</PackageReference>
<PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />
<PackageReference Include="Serilog.Settings.Configuration" Version="8.0.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="5.0.1">
<TreatAsUsed>true</TreatAsUsed>
</PackageReference>
<PackageReference Include="Serilog" Version="4.0.0"/>
<PackageReference Include="Serilog.Enrichers.Environment" Version="3.0.0"/>
<PackageReference Include="Serilog.Enrichers.Thread" Version="4.0.0"/>
<PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0"/>
<PackageReference Include="Serilog.Settings.Configuration" Version="8.0.1"/>
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0"/>
<PackageReference Include="Serilog.Sinks.File" Version="5.0.0"/>
</ItemGroup>
<ItemGroup>
<None Update="appsettings.json">
Expand Down
21 changes: 15 additions & 6 deletions KernelMemory.FileWatcher/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Polly.Contrib.WaitAndRetry;
using Polly.Extensions.Http;
using Serilog;
using Serilog.Events;

namespace KernelMemory.FileWatcher
{
Expand All @@ -16,7 +17,6 @@ public class Program
static async Task Main(string[] args)
{
using var host = CreateHostBuilder(args).Build();

StartWatcher(host.Services);

await host.RunAsync();
Expand All @@ -33,17 +33,20 @@ private static void StartWatcher(IServiceProvider services)

private static IHostBuilder CreateHostBuilder(string[] args)
{
Log.Logger = new LoggerConfiguration()
.MinimumLevel.Override("Microsoft", LogEventLevel.Information)
.Enrich.FromLogContext()
.WriteTo.Console()
.CreateBootstrapLogger();

var basePath = File.Exists("/config/appsettings.json") ? "/config" : AppContext.BaseDirectory;
var configuration = new ConfigurationBuilder()
.SetBasePath(basePath)
.AddJsonFile("appsettings.json", false)
.AddUserSecrets<Program>()
.AddEnvironmentVariables()
.Build();

Log.Logger = new LoggerConfiguration()
.ReadFrom.Configuration(configuration)
.CreateLogger();

return Host.CreateDefaultBuilder(args)
.ConfigureAppConfiguration(builder =>
{
Expand All @@ -52,9 +55,15 @@ private static IHostBuilder CreateHostBuilder(string[] args)
})
.ConfigureServices(services =>
{
services.AddSerilog((services, lc) => lc
.ReadFrom.Services(services)
.Enrich.FromLogContext()
.WriteTo.Console()
.WriteTo.File(Path.Exists("/config/logs") ? "/config/logs/km-filewatcher.log" : "logs/km-filewatcher.log", rollingInterval: RollingInterval.Day, retainedFileCountLimit: 14)
.MinimumLevel.Warning()
.ReadFrom.Configuration(configuration));
services.Configure<FileWatcherOptions>(configuration.GetSection("FileWatcher"));
services.Configure<KernelMemoryOptions>(configuration.GetSection("KernelMemory"));
services.AddLogging(c => c.AddSerilog().AddConsole());
services.AddSingleton<IMessageStore, MessageStore>();
services.AddHttpClient("km-client", client =>
{
Expand Down
116 changes: 53 additions & 63 deletions KernelMemory.FileWatcher/Services/HttpWorker.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,15 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using System.Threading;
using Serilog.Data;

namespace KernelMemory.FileWatcher.Services
{
internal class HttpWorker : IHostedService, IDisposable
internal class HttpWorker : BackgroundService
{
private readonly ILogger<HttpWorker> logger;
private readonly IHttpClientFactory httpClientFactory;
private readonly IMessageStore store;
private readonly KernelMemoryOptions options;
private PeriodicTimer? timer;

public HttpWorker(ILogger<HttpWorker> logger, IOptions<KernelMemoryOptions> options, IHttpClientFactory httpClientFactory, IMessageStore messageStore)
{
Expand All @@ -24,79 +21,72 @@ public HttpWorker(ILogger<HttpWorker> logger, IOptions<KernelMemoryOptions> opti
this.store = messageStore;
}

public async Task StartAsync(CancellationToken cancellationToken)
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
logger.LogInformation("Starting HttpWorker");
timer = new PeriodicTimer(options.Schedule);
while (await timer.WaitForNextTickAsync(cancellationToken))
this.logger.LogInformation("Starting HttpWorker");

using PeriodicTimer timer = new PeriodicTimer(options.Schedule);
try
{
var messages = store.TakeAll();
if (messages.Any())
while (await timer.WaitForNextTickAsync(stoppingToken) && !stoppingToken.IsCancellationRequested)
{
var tasks = new List<Action>();

foreach (var message in messages)
var messages = store.TakeAll();
if (messages.Any())
{
tasks.Add(async () =>
var parallelOptions = new ParallelOptions
{
if (message.Event?.EventType != FileEventType.Ignore)
{
logger.LogInformation($"Processing message {message.DocumentId} for file {message.Event?.FileName} of type {message.Event?.EventType}");
var client = httpClientFactory.CreateClient("km-client");
string endpoint;
HttpResponseMessage? response = null;

if (message.Event is { EventType: FileEventType.Upsert })
{
endpoint = "/upload";
CancellationToken = stoppingToken,
MaxDegreeOfParallelism = options.ParallelUploads
};
Parallel.Invoke(parallelOptions, messages.Select(message => (Action)(() => BuildMessageTask(message, stoppingToken))).ToArray());
}
else
{
logger.LogInformation("Nothing to process");

}
}
} catch (OperationCanceledException)
{
this.logger.LogInformation("Stopping HttpWorker");
}
}

var content = new MultipartFormDataContent();
var fileContent = new StreamContent(File.OpenRead(message.Event.Directory));
content.Add(fileContent, "file", message.Event.FileName);
content.Add(new StringContent(message.Index), "index");
content.Add(new StringContent(message.DocumentId), "documentid");
response = await client.PostAsync(endpoint, content);
}
else if (message.Event is { EventType: FileEventType.Delete })
{
endpoint = $"/documents?index={message.Index}&documentId={message.DocumentId}";
response = await client.DeleteAsync(endpoint);
}
private async Task BuildMessageTask(Message message, CancellationToken stoppingToken)
{
if (message.Event?.EventType != FileEventType.Ignore)
{
logger.LogInformation($"Processing message {message.DocumentId} for file {message.Event?.FileName} of type {message.Event?.EventType}");
var client = httpClientFactory.CreateClient("km-client");
string endpoint;
HttpResponseMessage? response = null;

if (response is { IsSuccessStatusCode: true })
{
logger.LogInformation($"Sent message {message.DocumentId} to {options.Endpoint}");
}
else
{
logger.LogError($"Failed to send message {message.DocumentId} to {options.Endpoint}");
}
}
});
}
if (message.Event is { EventType: FileEventType.Upsert })
{
endpoint = "/upload";

var parallelOptions = new ParallelOptions
{
CancellationToken = cancellationToken,
MaxDegreeOfParallelism = options.ParallelUploads
};
Parallel.Invoke(parallelOptions, tasks.ToArray());
var content = new MultipartFormDataContent();
var fileContent = new StreamContent(File.OpenRead(message.Event.Directory));
content.Add(fileContent, "file", message.Event.FileName);
content.Add(new StringContent(message.Index), "index");
content.Add(new StringContent(message.DocumentId), "documentid");
response = await client.PostAsync(endpoint, content, stoppingToken);
}
else if (message.Event is { EventType: FileEventType.Delete })
{
endpoint = $"/documents?index={message.Index}&documentId={message.DocumentId}";
response = await client.DeleteAsync(endpoint, stoppingToken);
}

if (response is { IsSuccessStatusCode: true })
{
logger.LogInformation($"Sent message {message.DocumentId} to {options.Endpoint}");
}
else
{
logger.LogInformation("Nothing to process");
logger.LogError($"Failed to send message {message.DocumentId} to {options.Endpoint}");
}
}
}

public Task StopAsync(CancellationToken cancellationToken)
{
return Task.CompletedTask;
}
public void Dispose()
{
timer?.Dispose();
}
}
}
19 changes: 4 additions & 15 deletions KernelMemory.FileWatcher/appsettings.json
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,20 @@
{
"Serilog": {
"Using": [ "Serilog.Sinks.Console" ],
"MinimumLevel": "Debug",
"WriteTo": [
{ "Name": "Console" }
]
"MinimumLevel": "Information"
},
"FileWatcher": {
"Directories": [
//{
// "Path": "G:\\tmp\\km-filewatcher\\folder_01",
// "Filter": "*.md",
// "Index": "folder-01",
// "IncludeSubdirectories": true,
// "InitialScan": true
//},
{
"Path": "G:\\tmp\\km-filewatcher\\folder_02",
"Path": "/tmp/tmp-docs/",
"Filter": "*.md",
"Index": "folder-02",
"Index": "tmp",
"IncludeSubdirectories": true,
"InitialScan": true
}
]
},
"KernelMemory": {
"Endpoint": "http://192.168.65.2:9001",
"Endpoint": "http://192.168.56.2:9001",
"Schedule": "00:00:30",
"Retries": 5,
"ParallelUploads": 5
Expand Down

0 comments on commit 59ede69

Please sign in to comment.