Only treat a file with directives as an FBP

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer.UnitTests/FileBasedProgramsWorkspaceTests.cs

@@ -125,4 +125,60 @@ void OtherMethod()
         // Should have the appropriate generated files now that we ran a design time build
         Assert.Contains(canonicalDocumentTwo.Project.Documents, d => d.Name == "Canonical.AssemblyInfo.cs");
     }
+
+    [Theory, CombinatorialData]
+    public async Task TestFileBecomesFileBasedProgramWhenDirectiveAdded(bool mutatingLspWorkspace)
+    {
+        // Create a server that supports LSP misc files and verify no misc files present.
+        await using var testLspServer = await CreateTestLspServerAsync(string.Empty, mutatingLspWorkspace, new InitializationOptions { ServerKind = WellKnownLspServerKinds.CSharpVisualBasicLspServer });
+        Assert.Null(await GetMiscellaneousDocumentAsync(testLspServer));
+
+        var looseFileUriOne = ProtocolConversions.CreateAbsoluteDocumentUri(@"C:\SomeFile.cs");
+        await testLspServer.OpenDocumentAsync(looseFileUriOne, """
+            Console.WriteLine("Hello World!");
+            """).ConfigureAwait(false);
+
+        // File should be initially added as a primordial document in the canonical misc files project with no metadata references.
+        var (miscFilesWorkspace, looseDocumentOne) = await GetRequiredLspWorkspaceAndDocumentAsync(looseFileUriOne, testLspServer).ConfigureAwait(false);
+        Assert.Equal(WorkspaceKind.MiscellaneousFiles, miscFilesWorkspace.Kind);
+        // Should have the primordial canonical document and the loose document.
+        Assert.Equal(2, looseDocumentOne.Project.Documents.Count());
+        Assert.Empty(looseDocumentOne.Project.MetadataReferences);
+
+        // Wait for the canonical project to finish loading.
+        await testLspServer.TestWorkspace.GetService<AsynchronousOperationListenerProvider>().GetWaiter(FeatureAttribute.Workspace).ExpeditedWaitAsync();
+
+        // Verify the document is loaded in the canonical project.
+        var (_, canonicalDocumentOne) = await GetRequiredLspWorkspaceAndDocumentAsync(looseFileUriOne, testLspServer).ConfigureAwait(false);
+        Assert.NotEqual(looseDocumentOne, canonicalDocumentOne);
+        // Should have the appropriate generated files now that we ran a design time build
+        Assert.Contains(canonicalDocumentOne.Project.Documents, d => d.Name == "Canonical.AssemblyInfo.cs");
+
+        // Adding a #! directive to a misc file causes it to move to a file-based program project.
+        var textToInsert = $"#!/usr/bin/env dotnet{Environment.NewLine}";
+        await testLspServer.InsertTextAsync(looseFileUriOne, (Line: 0, Column: 0, Text: textToInsert));
+        var (_, fileBasedDocumentOne) = await GetRequiredLspWorkspaceAndDocumentAsync(looseFileUriOne, testLspServer).ConfigureAwait(false);
+
+        // The document is now in a primordial state in the FileBasedProgramsProjectSystem.
+        Assert.NotEqual(fileBasedDocumentOne, canonicalDocumentOne);
+        var fileBasedProject = fileBasedDocumentOne.Project;
+        Assert.Same(miscFilesWorkspace, fileBasedProject.Solution.Workspace);
+        Assert.NotEqual(canonicalDocumentOne.Project.Id, fileBasedProject.Id);
+        Assert.Equal("""
+            #!/usr/bin/env dotnet
+            Console.WriteLine("Hello World!");
+            """,
+            (await fileBasedDocumentOne.GetSyntaxRootAsync())!.ToFullString());
+
+        // Verify that the project system remains in a good state, when intermediate requests come in while the file-based program project is still loaded.
+        var (_, alsoFileBasedDocumentOne) = await GetRequiredLspWorkspaceAndDocumentAsync(looseFileUriOne, testLspServer).ConfigureAwait(false);
+        Assert.Equal(fileBasedProject.Id, alsoFileBasedDocumentOne.Project.Id);
+
+        // Wait for the file-based program project to load.
+        await testLspServer.TestWorkspace.GetService<AsynchronousOperationListenerProvider>().GetWaiter(FeatureAttribute.Workspace).ExpeditedWaitAsync();
+        var (hostWorkspace, fullFileBasedDocumentOne) = await GetRequiredLspWorkspaceAndDocumentAsync(looseFileUriOne, testLspServer).ConfigureAwait(false);
+        Assert.Equal(WorkspaceKind.Host, hostWorkspace!.Kind);
+        Assert.NotEqual(fileBasedProject.Id, fullFileBasedDocumentOne!.Project.Id);
+        Assert.Contains(fullFileBasedDocumentOne!.Project.Documents, d => d.Name == "SomeFile.AssemblyInfo.cs");
+    }
 }

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer/FileBasedPrograms/FileBasedProgramsProjectSystem.cs

@@ -77,12 +77,22 @@ public async ValueTask<bool> IsMiscellaneousFilesDocumentAsync(TextDocument docu
         //   1.  The document is a primordial document (either not loaded yet or doesn't support design time build) - it will be in the misc files workspace.
         //   2.  The document is loaded as a canonical misc file - these are always in the misc files workspace.
         //   3.  The document is loaded as a file based program - then it will be in the main workspace where the project path matches the source file path.
+
+        // NB: The FileBasedProgramsProjectSystem uses the document file path (the on-disk path) as the projectPath in 'IsProjectLoadedAsync'.
+        var isLoadedAsFileBasedProgram = document.FilePath is { } filePath && await IsProjectLoadedAsync(filePath, cancellationToken);
+
+        // If this document has a file-based program syntactic marker, but we aren't loading it in a file-based programs project,
+        // we need the caller to remove and re-add this document, so that it gets put in a file-based programs project instead.
+        if (!isLoadedAsFileBasedProgram && VirtualProjectXmlProvider.IsFileBasedProgram(await document.GetTextAsync(cancellationToken)))
+            return false;
+
         if (document.Project.Solution.Workspace == _workspaceFactory.MiscellaneousFilesWorkspaceProjectFactory.Workspace)
             return true;
 
-        if (document.Project.FilePath is not null && await IsProjectLoadedAsync(document.Project.FilePath, cancellationToken))
+        if (isLoadedAsFileBasedProgram)
             return true;
 
+        // Document is not managed by this project system. Caller should unload it.
         return false;
     }
 
@@ -102,7 +112,7 @@ public async ValueTask<bool> IsMiscellaneousFilesDocumentAsync(TextDocument docu
         if (supportsDesignTimeBuild)
         {
             // For virtual (non-file) URIs or non-file-based programs, use the canonical loader
-            if (uri.ParsedUri is null || !uri.ParsedUri.IsFile || !VirtualProjectXmlProvider.IsFileBasedProgram(documentFilePath, documentText))
+            if (uri.ParsedUri is null || !uri.ParsedUri.IsFile || !VirtualProjectXmlProvider.IsFileBasedProgram(documentText))
             {
                 return await _canonicalMiscFilesLoader.AddMiscellaneousDocumentAsync(documentFilePath, documentText, CancellationToken.None);
             }
@@ -171,23 +181,18 @@ public async ValueTask<bool> TryRemoveMiscellaneousDocumentAsync(DocumentUri uri
         // When loading a virtual project, the path to the on-disk source file is not used. Instead the path is adjusted to end with .csproj.
         // This is necessary in order to get msbuild to apply the standard c# props/targets to the project.
         var virtualProjectPath = VirtualProjectXmlProvider.GetVirtualProjectPath(documentPath);
-
-        var loader = _workspaceFactory.MiscellaneousFilesWorkspaceProjectFactory.CreateFileTextLoader(documentPath);
-        var textAndVersion = await loader.LoadTextAsync(new LoadTextOptions(SourceHashAlgorithms.Default), cancellationToken);
-        var isFileBasedProgram = VirtualProjectXmlProvider.IsFileBasedProgram(documentPath, textAndVersion.Text);
-
         const BuildHostProcessKind buildHostKind = BuildHostProcessKind.NetCore;
         var buildHost = await buildHostProcessManager.GetBuildHostAsync(buildHostKind, virtualProjectPath, dotnetPath: null, cancellationToken);
         var loadedFile = await buildHost.LoadProjectAsync(virtualProjectPath, virtualProjectContent, languageName: LanguageNames.CSharp, cancellationToken);
 
         return new RemoteProjectLoadResult
         {
             ProjectFile = loadedFile,
-            // If it's a proper file based program, we'll put it in the main host workspace factory since we want cross-project references to work.
-            // Otherwise, we'll keep it in miscellaneous files.
-            ProjectFactory = isFileBasedProgram ? _workspaceFactory.HostProjectFactory : _workspaceFactory.MiscellaneousFilesWorkspaceProjectFactory,
-            IsFileBasedProgram = isFileBasedProgram,
-            IsMiscellaneousFile = !isFileBasedProgram,
+            // If we have made it this far, we must have determined that the document is a file-based program.
+            // TODO: we should assert this somehow. However, we cannot use the on-disk state of the file to do so, because the decision to load this as a file-based program was based on in-editor content.
+            ProjectFactory = _workspaceFactory.HostProjectFactory,
+            IsFileBasedProgram = true,
+            IsMiscellaneousFile = false,
             PreferredBuildHostKind = buildHostKind,
             ActualBuildHostKind = buildHostKind,
         };

src/LanguageServer/Microsoft.CodeAnalysis.LanguageServer/FileBasedPrograms/VirtualProjectXmlProvider.cs

@@ -4,6 +4,7 @@
 
 using System.Collections.Immutable;
 using System.Composition;
+using System.Diagnostics.CodeAnalysis;
 using System.Runtime.InteropServices;
 using System.Security;
 using System.Security.Cryptography;
@@ -92,17 +93,30 @@ internal class VirtualProjectXmlProvider(DotnetCliHelper dotnetCliHelper)
     /// Adjusts a path to a file-based program for use in passing the virtual project to msbuild.
     /// (msbuild needs the path to end in .csproj to recognize as a C# project and apply all the standard props/targets to it.)
     /// </summary>
-    internal static string GetVirtualProjectPath(string documentFilePath)
+    [return: NotNullIfNotNull(nameof(documentFilePath))]
+    internal static string? GetVirtualProjectPath(string? documentFilePath)
         => Path.ChangeExtension(documentFilePath, ".csproj");
 
-    internal static bool IsFileBasedProgram(string documentFilePath, SourceText text)
+    /// <summary>
+    /// Indicates whether the editor considers the text to be a file-based program.
+    /// If this returns false, the text is either a miscellaneous file or is part of an ordinary project.
+    /// </summary>
+    /// <remarks>
+    /// The editor considers the text to be a file-based program if it has any '#!' or '#:' directives at the top.
+    /// Note that a file with top-level statements but no directives can still work with 'dotnet app.cs' etc. on the CLI, but will be treated as a misc file in the editor.
+    /// </remarks>
+    internal static bool IsFileBasedProgram(SourceText text)
     {
-        // https://github.com/dotnet/roslyn/issues/78878: this needs to be adjusted to be more sustainable.
-        // When we adopt the dotnet run-api, we need to get rid of this or adjust it to be more sustainable (e.g. using the appropriate document to get a syntax tree)
-        var tree = CSharpSyntaxTree.ParseText(text, options: CSharpParseOptions.Default.WithLanguageVersion(LanguageVersion.Preview), path: documentFilePath);
-        var root = tree.GetRoot();
-        var isFileBasedProgram = root.GetLeadingTrivia().Any(SyntaxKind.IgnoredDirectiveTrivia) || root.ChildNodes().Any(node => node.IsKind(SyntaxKind.GlobalStatement));
-        return isFileBasedProgram;
+        var tokenizer = SyntaxFactory.CreateTokenParser(text, CSharpParseOptions.Default.WithFeatures([new("FileBasedProgram", "true")]));
+        var result = tokenizer.ParseLeadingTrivia();
+        var triviaList = result.Token.LeadingTrivia;
+        foreach (var trivia in triviaList)
+        {
+            if (trivia.Kind() is SyntaxKind.ShebangDirectiveTrivia or SyntaxKind.IgnoredDirectiveTrivia)
+                return true;
+        }
+
+        return false;
     }
 
     #region Temporary copy of subset of dotnet run-api behavior for fallback: https://github.com/dotnet/roslyn/issues/78618

src/LanguageServer/Protocol.TestUtilities/AbstractLspMiscellaneousFilesWorkspaceTests.cs

@@ -377,6 +377,14 @@ public async Task TestLspTransfersFromMiscellaneousFilesToHostWorkspaceAsync(boo
         return (workspace, document as Document);
     }
 
+    private protected static async Task<(Workspace workspace, Document document)> GetRequiredLspWorkspaceAndDocumentAsync(DocumentUri uri, TestLspServer testLspServer)
+    {
+        var (workspace, document) = await GetLspWorkspaceAndDocumentAsync(uri, testLspServer);
+        Assert.NotNull(workspace);
+        Assert.NotNull(document);
+        return (workspace, document);
+    }
+
     private protected static async ValueTask<Document?> GetMiscellaneousDocumentAsync(TestLspServer testLspServer)
     {
         var documents = await testLspServer.GetManagerAccessor().GetMiscellaneousDocumentsAsync(static p => p.Documents).ToImmutableArrayAsync(CancellationToken.None);