Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@
<PackageVersion Include="KubernetesClient" Version="19.0.2" />
<PackageVersion Include="JsonPatch.Net" Version="5.0.2" />
<PackageVersion Include="Markdig" Version="1.2.0" />
<PackageVersion Include="Microsoft.AI.Foundry.Local" Version="0.3.0" />
<PackageVersion Include="Microsoft.AI.Foundry.Local" Version="1.1.0" />
<PackageVersion Include="Microsoft.Agents.AI.OpenAI" Version="1.5.0" />
<PackageVersion Include="Microsoft.Data.SqlClient" Version="7.0.1" />
<PackageVersion Include="Microsoft.FluentUI.AspNetCore.Components" Version="4.14.1" />
Expand Down
4 changes: 2 additions & 2 deletions src/Aspire.Hosting.Foundry/FoundryDeploymentResource.cs
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ public FoundryDeploymentResource(string name, string modelName, string modelVers
}

/// <summary>
/// This field is used to store the model id that is downloaded by Foundry Local based on the local machine's GPU.
/// It is used in the connection string instead of <see cref="ModelName" />.
/// This field is used to store the model identifier used by Foundry Local.
/// It is used in the connection string instead of <see cref="ModelName"/>.
/// </summary>
internal string? ModelId { get; set; }

Expand Down
118 changes: 57 additions & 61 deletions src/Aspire.Hosting.Foundry/FoundryExtensions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,12 @@
using Azure.Provisioning.CognitiveServices;
using Azure.Provisioning.Expressions;
using Azure.Provisioning.Resources;
using Microsoft.AI.Foundry.Local;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using Microsoft.Extensions.Diagnostics.HealthChecks;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using System.Threading.Channels;
using static Azure.Provisioning.Expressions.BicepFunction;

namespace Aspire.Hosting;
Expand Down Expand Up @@ -174,9 +176,8 @@ public static IResourceBuilder<FoundryResource> RunAsFoundryLocal(this IResource
ThrowIfProjectsConfiguredForLocal(builder, resource);
resource.Annotations.Add(new EmulatorResourceAnnotation());

builder.ApplicationBuilder.Services.AddSingleton<FoundryLocalManager>();

builder.WithInitializer();
builder.OnResourceStopped(static (_, _, ct) => FoundryLocalService.StopAsync(ct));

foreach (var deployment in resource.Deployments)
{
Expand All @@ -190,11 +191,12 @@ public static IResourceBuilder<FoundryResource> RunAsFoundryLocal(this IResource
builder.ApplicationBuilder.Services.AddHealthChecks()
.Add(new HealthCheckRegistration(
healthCheckKey,
sp => new FoundryLocalHealthCheck(sp.GetRequiredService<FoundryLocalManager>()),
sp => new FoundryLocalHealthCheck(),
failureStatus: default,
tags: default,
timeout: default
));
builder.ApplicationBuilder.Services.TryAddEnumerable(ServiceDescriptor.Singleton<IHostedService, FoundryLocalLifecycleService>());

builder.WithHealthCheck(healthCheckKey);

Expand Down Expand Up @@ -286,10 +288,9 @@ private static IResourceBuilder<FoundryResource> WithInitializer(this IResourceB
=> Task.Run(async () =>
{
var rns = @event.Services.GetRequiredService<ResourceNotificationService>();
var manager = @event.Services.GetRequiredService<FoundryLocalManager>();
var logger = @event.Services.GetRequiredService<ResourceLoggerService>().GetLogger(resource);

resource.ApiKey = manager.ApiKey;
resource.ApiKey = FoundryLocalService.ApiKey;

await rns.PublishUpdateAsync(resource, state => state with
{
Expand All @@ -298,16 +299,16 @@ await rns.PublishUpdateAsync(resource, state => state with

try
{
await manager.StartServiceAsync(ct).ConfigureAwait(false);
await FoundryLocalService.StartAsync(logger, ct).ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogInformation("Foundry Local could not be started. Ensure it's installed correctly: https://learn.microsoft.com/azure/ai-foundry/foundry-local/get-started (Error: {Error}).", e.Message);
}

if (manager.IsServiceRunning)
if (FoundryLocalService.IsServiceRunning)
{
resource.EmulatorServiceUri = manager.Endpoint;
resource.EmulatorServiceUri = FoundryLocalService.Endpoint;

await rns.PublishUpdateAsync(resource, state => state with
{
Expand Down Expand Up @@ -340,7 +341,6 @@ internal static IResourceBuilder<FoundryDeploymentResource> AsLocalDeployment(th
var rns = @event.Services.GetRequiredService<ResourceNotificationService>();
var loggerService = @event.Services.GetRequiredService<ResourceLoggerService>();
var logger = loggerService.GetLogger(deployment);
var manager = @event.Services.GetRequiredService<FoundryLocalManager>();
var eventing = @event.Services.GetRequiredService<IDistributedApplicationEventing>();

var model = deployment.ModelName;
Expand All @@ -353,67 +353,63 @@ await rns.PublishUpdateAsync(deployment, state => state with
Properties = [.. state.Properties, new(CustomResourceKnownProperties.Source, model)]
}).ConfigureAwait(false);

var result = manager.DownloadModelWithProgressAsync(model, ct: ct);
var progressChannel = Channel.CreateUnbounded<float>();
var downloadTask = DownloadModelAsync();

await foreach (var progress in result.ConfigureAwait(false))
await foreach (var progress in progressChannel.Reader.ReadAllAsync(ct).ConfigureAwait(false))
{
if (progress.IsCompleted && progress.ModelInfo is not null)
logger.LogInformation("Downloading model {Model}: {Progress:F2}%", model, progress);
await rns.PublishUpdateAsync(deployment, state => state with
{
// Set the model id that was actually downloaded. This is the value that is used in the
// connection string
State = new ResourceStateSnapshot($"Downloading model {model}: {progress:F2}%", KnownResourceStateStyles.Info)
}).ConfigureAwait(false);
}

deployment.ModelId = progress.ModelInfo.ModelId;
logger.LogInformation("Model {Model} downloaded successfully ({ModelId}).", model, deployment.ModelId);
try
{
deployment.ModelId = await downloadTask.ConfigureAwait(false);
logger.LogInformation("Model {Model} downloaded successfully ({ModelId}).", model, deployment.ModelId);

// Re-publish the connection string since the model id is now known
var connectionStringAvailableEvent = new ConnectionStringAvailableEvent(deployment, @event.Services);
await eventing.PublishAsync(connectionStringAvailableEvent, ct).ConfigureAwait(false);
// Re-publish the connection string since the model id is now known.
var connectionStringAvailableEvent = new ConnectionStringAvailableEvent(deployment, @event.Services);
await eventing.PublishAsync(connectionStringAvailableEvent, ct).ConfigureAwait(false);

await rns.PublishUpdateAsync(deployment, state => state with
{
Properties = [.. state.Properties, new(CustomResourceKnownProperties.Source, $"{model} ({deployment.ModelId})")]
}).ConfigureAwait(false);
await rns.PublishUpdateAsync(deployment, state => state with
{
Properties = [.. state.Properties, new(CustomResourceKnownProperties.Source, $"{model} ({deployment.ModelId})")]
}).ConfigureAwait(false);

await rns.PublishUpdateAsync(deployment, state => state with
{
State = new ResourceStateSnapshot("Loading model", KnownResourceStateStyles.Info)
}).ConfigureAwait(false);
await rns.PublishUpdateAsync(deployment, state => state with
{
State = new ResourceStateSnapshot("Loading model", KnownResourceStateStyles.Info)
}).ConfigureAwait(false);

try
{
_ = await manager.LoadModelAsync(deployment.ModelId, ct: ct).ConfigureAwait(false);

await rns.PublishUpdateAsync(deployment, state => state with
{
State = KnownResourceStates.Running
}).ConfigureAwait(false);
}
catch (Exception e)
{
// LoadModelAsync throws IOE when the model is invalid.
logger.LogInformation("Failed to start {Model}. Error: {Error}", model, e.Message);

await rns.PublishUpdateAsync(deployment, state => state with
{
State = KnownResourceStates.FailedToStart
}).ConfigureAwait(false);
}
}
else if (progress.IsCompleted && !string.IsNullOrEmpty(progress.ErrorMessage))
await FoundryLocalService.LoadModelAsync(deployment.ModelId, ct).ConfigureAwait(false);

await rns.PublishUpdateAsync(deployment, state => state with
{
logger.LogInformation("Failed to start {Model}. Error: {Error}", model, progress.ErrorMessage);
await rns.PublishUpdateAsync(deployment, state => state with
{
State = KnownResourceStates.FailedToStart
}).ConfigureAwait(false);
State = KnownResourceStates.Running
}).ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogInformation("Failed to start {Model}. Error: {Error}", model, e.Message);

await rns.PublishUpdateAsync(deployment, state => state with
{
State = KnownResourceStates.FailedToStart
}).ConfigureAwait(false);
}

async Task<string> DownloadModelAsync()
{
try
{
return await FoundryLocalService.DownloadModelAsync(model, progress => progressChannel.Writer.TryWrite(progress), ct).ConfigureAwait(false);
}
else
finally
{
logger.LogInformation("Downloading model {Model}: {Progress:F2}%", model, progress.Percentage);
await rns.PublishUpdateAsync(deployment, state => state with
{
State = new ResourceStateSnapshot($"Downloading model {model}: {progress.Percentage:F2}%", KnownResourceStateStyles.Info)
}).ConfigureAwait(false);
progressChannel.Writer.TryComplete();
}
}
}, ct);
Expand All @@ -426,7 +422,7 @@ await rns.PublishUpdateAsync(deployment, state => state with
builder.ApplicationBuilder.Services.AddHealthChecks()
.Add(new HealthCheckRegistration(
healthCheckKey,
sp => new LocalModelHealthCheck(modelId: deployment.ModelId, sp.GetRequiredService<FoundryLocalManager>()),
sp => new LocalModelHealthCheck(modelId: deployment.ModelId),
failureStatus: default,
tags: default,
timeout: default
Expand Down
5 changes: 2 additions & 3 deletions src/Aspire.Hosting.Foundry/FoundryLocalHealthCheck.cs
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using Microsoft.AI.Foundry.Local;
using Microsoft.Extensions.Diagnostics.HealthChecks;

namespace Aspire.Hosting.Foundry;

internal sealed class FoundryLocalHealthCheck(FoundryLocalManager manager) : IHealthCheck
internal sealed class FoundryLocalHealthCheck : IHealthCheck
{
public Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default)
{
if (!manager.IsServiceRunning)
if (!FoundryLocalService.IsServiceRunning)
{
return Task.FromResult(HealthCheckResult.Unhealthy("Foundry Local not running"));
}
Expand Down
Loading
Loading