Support for project references (#165)

This adds support for detecting project references when reading through the binlog. This is mostly based on matching a compiler call with its output binaries (by MVID) and then re-hydrating project references based off of that. 

closes #24

README.md

@@ -1,5 +1,4 @@
-Compiler Logs
-===
+# Compiler Logs
 
 [![codecov](https://codecov.io/gh/jaredpar/complog/graph/badge.svg?token=MIM7Y2JZ5G)](https://codecov.io/gh/jaredpar/complog)
 
@@ -70,7 +69,9 @@ When trying to get a compiler log from a build that occurs in a GitHub action yo
 ```
 
 ## Debugging Compiler Logs
+
 ### Running locally
+
 To re-run all of the compilations in a compiler log use the `replay` command
 
 ```cmd
@@ -83,7 +84,8 @@ Microsoft.CodeAnalysis.XunitHook.csproj (net472) ...Success
 Passing the `-export` argument will cause all failed compilations to be exported to the local disk for easy analysis.
 
 ### Debugging in Visual Studio
-To debug a compilation in Visual Studio first export it to disk: 
+
+To debug a compilation in Visual Studio first export it to disk:
 
 ```cmd
 > complog export build.complog

notes.txt

@@ -0,0 +1,16 @@
+ Converting looks like this       
+
+        public static PortableExecutableReference EmitToPortableExecutableReference(
+            this Compilation comp,
+            EmitOptions options = null,
+            bool embedInteropTypes = false,
+            ImmutableArray<string> aliases = default,
+            DiagnosticDescription[] expectedWarnings = null)
+        {
+            var image = comp.EmitToArray(options, expectedWarnings: expectedWarnings);
+            if (comp.Options.OutputKind == OutputKind.NetModule)
+            {
+                return ModuleMetadata.CreateFromImage(image).GetReference(display: comp.MakeSourceModuleName());
+            }
+            else
+            {

src/Basic.CompilerLog.UnitTests/Basic.CompilerLog.UnitTests.csproj

@@ -42,6 +42,9 @@
     <EmbeddedResource Include="Resources\MetadataVersion1\console.complog">
         <LogicalName>MetadataVersion1.console.complog</LogicalName>
     </EmbeddedResource>
+    <EmbeddedResource Include="Resources\MetadataVersion2\console.complog">
+        <LogicalName>MetadataVersion2.console.complog</LogicalName>
+    </EmbeddedResource>
     <EmbeddedResource Include="Resources\linux-console.complog">
         <LogicalName>linux-console.complog</LogicalName>
     </EmbeddedResource>

src/Basic.CompilerLog.UnitTests/CompilerLogFixture.cs

@@ -57,6 +57,11 @@ public sealed class CompilerLogFixture : FixtureBase, IDisposable
 
     internal Lazy<LogData> ConsoleNoGenerator { get; }
 
+    /// <summary>
+    /// A console project that has a reference to a library
+    /// </summary>
+    internal Lazy<LogData> ConsoleWithReference { get; }
+
     /// <summary>
     /// This is a console project that has every nasty feature that can be thought of
     /// like resources, line directives, embeds, etc ... Rather than running a 
@@ -336,6 +341,45 @@ partial class Util {
             RunDotnetCommand("build -bl -nr:false", scratchPath);
         });
 
+        ConsoleWithReference = WithBuild("console-with-project-ref..complog", void (string scratchPath) =>
+        {
+            RunDotnetCommand("new sln -n ConsoleWithProjectRef", scratchPath);
+
+            // Create a class library for referencing
+            var classLibPath = Path.Combine(scratchPath, "classlib");
+            _ = Directory.CreateDirectory(classLibPath);
+            RunDotnetCommand("new classlib -o . -n util", classLibPath);
+            File.WriteAllText(
+                Path.Combine(classLibPath, "Class1.cs"),
+                """
+                using System;
+                namespace Util;
+                public static class NameInfo
+                {
+                    public static string GetName() => "Hello World";
+                }
+                """,
+                TestBase.DefaultEncoding);
+            RunDotnetCommand($@"sln add ""{classLibPath}""", scratchPath);
+
+            // Create a console project that references the class library
+            var consolePath = Path.Combine(scratchPath, "console");
+            _ = Directory.CreateDirectory(consolePath);
+            RunDotnetCommand("new console -o . -n console-with-reference", consolePath);
+            File.WriteAllText(
+                Path.Combine(consolePath, "Program.cs"),
+                """
+                using System;
+                using Util;
+                Console.WriteLine(NameInfo.GetName());
+                """,
+                TestBase.DefaultEncoding);
+            RunDotnetCommand($@"add . reference ""{classLibPath}""", consolePath);
+            RunDotnetCommand($@"sln add ""{consolePath}""", scratchPath);
+
+            RunDotnetCommand("build -bl -nr:false", scratchPath);
+        });
+
         if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
         {
             WpfApp = WithBuild("wpfapp.complog", void (string scratchPath) =>

src/Basic.CompilerLog.UnitTests/CompilerLogReaderTests.cs

@@ -440,7 +440,7 @@ void Core(string contentFilePath)
 
     [Theory]
     [InlineData("MetadataVersion1.console.complog")]
-    public void MetadataCompat(string resourceName)
+    public void MetadataCompatV1(string resourceName)
     {
         if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
         {
@@ -449,6 +449,18 @@ public void MetadataCompat(string resourceName)
         }
     }
 
+    [Theory]
+    [InlineData("MetadataVersion2.console.complog")]
+    public void MetadataCompatV2(string resourceName)
+    {
+        using var stream = ResourceLoader.GetResourceStream(resourceName);
+        using var reader = CompilerLogReader.Create(stream);
+        foreach (var compilerCall in reader.ReadAllCompilerCalls())
+        {
+            Assert.NotNull(compilerCall.ProjectFileName);
+        }
+    }
+
     [Fact]
     public void Disposed()
     {
@@ -465,4 +477,81 @@ public void VisualBasic()
         Assert.True(data.IsVisualBasic);
         Assert.True(data.CompilerCall.IsVisualBasic);
     }
+
+    [Fact]
+    public void ProjectReferences_Simple()
+    {
+        using var reader = CompilerLogReader.Create(Fixture.Console.Value.CompilerLogPath);
+        var compilerCall = reader.ReadCompilerCall(0);
+        var arguments = BinaryLogUtil.ReadCommandLineArgumentsUnsafe(compilerCall);
+        var (assemblyFilePath, refAssemblyFilePath) = RoslynUtil.GetAssemblyOutputFilePaths(arguments);
+        AssertProjectRef(assemblyFilePath);
+        AssertProjectRef(refAssemblyFilePath);
+
+        void AssertProjectRef(string? filePath)
+        {
+            Assert.NotNull(filePath);
+            var mvid = RoslynUtil.ReadMvid(filePath);
+            Assert.True(reader.TryGetCompilerCallIndex(mvid, out var callIndex));
+            Assert.Equal(reader.GetIndex(compilerCall), callIndex);
+        }
+    }
+
+    [Fact]
+    public void ProjectReferences_ReadReference()
+    {
+        using var reader = CompilerLogReader.Create(Fixture.ConsoleWithReference.Value.CompilerLogPath);
+        var classLibCompilerCall = reader
+            .ReadAllCompilerCalls(cc => cc.ProjectFileName == "util.csproj")
+            .Single();
+        var consoleCompilerCall = reader
+            .ReadAllCompilerCalls(cc => cc.ProjectFileName == "console-with-reference.csproj")
+            .Single();
+        var count = 0;
+        foreach (var rawReferenceData in reader.ReadAllReferenceData(consoleCompilerCall))
+        {
+            if (reader.TryGetCompilerCallIndex(rawReferenceData.Mvid, out var callIndex))
+            {
+                Assert.Equal(reader.GetIndex(classLibCompilerCall), callIndex);
+                count++;
+            }
+        }
+        Assert.Equal(1, count);
+    }
+
+    [Fact]
+    public void ProjectReferences_Corrupted()
+    {
+        RunDotNet($"new console --name example --output .", Root.DirectoryPath);
+        RunDotNet("build -bl -nr:false", Root.DirectoryPath);
+        var binlogFilePath = Path.Combine(Root.DirectoryPath, "msbuild.binlog");
+
+        var mvidList = CorruptAssemblies();
+        Assert.NotEmpty(mvidList);
+        using var reader = CompilerLogReader.Create(binlogFilePath);
+        foreach (var mvid in mvidList)
+        {
+            Assert.False(reader.TryGetCompilerCallIndex(mvid, out _));
+        }
+
+        List<Guid> CorruptAssemblies()
+        {
+            var list = new List<Guid>();
+            using var binlogReader = BinaryLogReader.Create(binlogFilePath);
+            foreach (var compilerCall in binlogReader.ReadAllCompilerCalls())
+            {
+                var (assemblyPath, refAssemblyPath) = RoslynUtil.GetAssemblyOutputFilePaths(BinaryLogUtil.ReadCommandLineArgumentsUnsafe(compilerCall));
+                Assert.NotNull(assemblyPath);
+                Assert.NotNull(refAssemblyPath);
+                list.Add(RoslynUtil.ReadMvid(assemblyPath));
+                list.Add(RoslynUtil.ReadMvid(refAssemblyPath));
+
+                File.WriteAllText(assemblyPath, "hello");
+                File.WriteAllText(refAssemblyPath, "hello ref");
+
+            }
+
+            return list;
+        }
+    }
 }

src/Basic.CompilerLog.UnitTests/FixtureBase.cs

@@ -30,6 +30,9 @@ protected void RunDotnetCommand(string args, string workingDirectory)
         diagnosticBuilder.AppendLine($"Standard Error: {result.StandardError}");
         diagnosticBuilder.AppendLine($"Finished: {(DateTime.UtcNow - start).TotalSeconds:F2}s");
         MessageSink.OnMessage(new DiagnosticMessage(diagnosticBuilder.ToString()));
-        Assert.True(result.Succeeded);
+        if (!result.Succeeded)
+        {
+            Assert.Fail($"Command failed: {diagnosticBuilder.ToString()}");
+        }
     }
 }

src/Basic.CompilerLog.UnitTests/SolutionReaderTests.cs

@@ -14,6 +14,7 @@ namespace Basic.CompilerLog.UnitTests;
 [Collection(CompilerLogCollection.Name)]
 public sealed class SolutionReaderTests : TestBase
 {
+    public List<SolutionReader> ReaderList { get; } = new();
     public CompilerLogFixture Fixture { get; }
 
     public SolutionReaderTests(ITestOutputHelper testOutputHelper, CompilerLogFixture fixture)
@@ -22,13 +23,29 @@ public SolutionReaderTests(ITestOutputHelper testOutputHelper, CompilerLogFixtur
         Fixture = fixture;
     }
 
+    public override void Dispose()
+    {
+        base.Dispose();
+        foreach (var reader in ReaderList)
+        {
+            reader.Dispose();
+        }
+    }
+
+    private Solution GetSolution(string compilerLogFilePath, BasicAnalyzerKind basicAnalyzerKind)
+    {
+        var reader = SolutionReader.Create(compilerLogFilePath, basicAnalyzerKind);
+        ReaderList.Add(reader);
+        var workspace = new AdhocWorkspace();
+        var solution = workspace.AddSolution(reader.ReadSolutionInfo());
+        return solution;
+    }
+
     [Theory]
     [CombinatorialData]
     public async Task DocumentsGeneratedDefaultHost(BasicAnalyzerKind basicAnalyzerKind)
     {
-        using var reader = SolutionReader.Create(Fixture.Console.Value.CompilerLogPath, basicAnalyzerKind);
-        var workspace = new AdhocWorkspace();
-        var solution = workspace.AddSolution(reader.ReadSolutionInfo());
+        var solution = GetSolution(Fixture.Console.Value.CompilerLogPath, basicAnalyzerKind);
         var project = solution.Projects.Single();
         Assert.NotEmpty(project.AnalyzerReferences);
         var docs = project.Documents.ToList();
@@ -48,4 +65,21 @@ public void CreateRespectLeaveOpen()
         // Throws if the underlying stream is disposed
         stream.Seek(0, SeekOrigin.Begin);
     }
+
+    [Fact]
+    public async Task ProjectReference_Simple()
+    {
+        var solution = GetSolution(Fixture.ConsoleWithReference.Value.CompilerLogPath, BasicAnalyzerKind.None);
+        var consoleProject = solution.Projects
+            .Where(x => x.Name == "console-with-reference.csproj")
+            .Single();
+        var projectReference = consoleProject.ProjectReferences.Single();
+        var utilProject = solution.GetProject(projectReference.ProjectId);
+        Assert.NotNull(utilProject);
+        Assert.Equal("util.csproj", utilProject.Name);
+        var compilation = await consoleProject.GetCompilationAsync();
+        Assert.NotNull(compilation);
+        var result = compilation.EmitToMemory();
+        Assert.True(result.Success);
+    }
 }

src/Basic.CompilerLog.Util/CommonUtil.cs

@@ -17,6 +17,7 @@ internal static class CommonUtil
 {
     internal const string MetadataFileName = "metadata.txt";
     internal const string AssemblyInfoFileName = "assemblyinfo.txt";
+    internal const string LogInfoFileName = "loginfo.txt";
     internal static readonly Encoding ContentEncoding = Encoding.UTF8;
     internal static readonly MessagePackSerializerOptions SerializerOptions = MessagePackSerializerOptions.Standard.WithAllowAssemblyVersionMismatch(true);
 

src/Basic.CompilerLog.Util/CompilerLogBuilder.cs

@@ -34,10 +34,11 @@ public bool Return(MemoryStream stream)
         }
     }
 
-    private readonly Dictionary<Guid, (string FileName, AssemblyName AssemblyName)> _mvidToRefInfoMap = new();
+    private readonly Dictionary<Guid, (string FileName, string AssemblyName)> _mvidToRefInfoMap = new();
     private readonly Dictionary<string, (Guid Mvid, string? AssemblyName, string? AssemblyInformationVersion)> _assemblyPathToMvidMap = new(PathUtil.Comparer);
     private readonly HashSet<string> _contentHashMap = new(PathUtil.Comparer);
     private readonly Dictionary<string, (string AssemblyName, string? CommitHash)> _compilerInfoMap = new(PathUtil.Comparer);
+    private readonly List<(int CompilerCallIndex, bool IsRefAssembly, Guid Mvid)> _compilerCallMvidList = new();
     private readonly DefaultObjectPool<MemoryStream> _memoryStreamPool = new(new MemoryStreamPoolPolicy(), maximumRetained: 5);
 
     private int _compilationCount;
@@ -103,6 +104,7 @@ string AddCompilationDataPack(CommandLineArguments commandLineArguments)
             AddContentIf(dataPack, RawContentKind.Win32Icon, commandLineArguments.Win32Icon);
             AddContentIf(dataPack, RawContentKind.Win32Manifest, commandLineArguments.Win32Manifest);
             AddContentIf(dataPack, RawContentKind.CryptoKeyFile, commandLineArguments.CompilationOptions.CryptoKeyFile);
+            AddAssemblyMvid(commandLineArguments);
             return WriteContentMessagePack(dataPack);
         }
 
@@ -174,6 +176,28 @@ void AddCompilationOptions(CompilationInfoPack infoPack, CommandLineArguments ar
                     MessagePackUtil.CreateVisualBasicCompilationOptionsPack((VisualBasicCompilationOptions)args.CompilationOptions));
             }
         }
+
+        void AddAssemblyMvid(CommandLineArguments args)
+        {
+            var (assemblyFilePath, refAssemblyFilePath) = RoslynUtil.GetAssemblyOutputFilePaths(args);
+            AddIf(assemblyFilePath, false);
+            AddIf(refAssemblyFilePath, false);
+            void AddIf(string? filePath, bool isRefAssembly)
+            {
+                if (filePath is not null && File.Exists(filePath))
+                {
+                    try
+                    {
+                        var mvid = RoslynUtil.ReadMvid(filePath);
+                        _compilerCallMvidList.Add((_compilationCount, isRefAssembly, mvid));
+                    }
+                    catch (Exception ex)
+                    {
+                        Diagnostics.Add($"Could not read emit assembly MVID for {filePath}: {ex.Message}");
+                    }
+                }
+            }
+        }
     }
 
     /// <summary>
@@ -243,7 +267,7 @@ public void Close()
         try
         {
             WriteMetadata();
-            WriteAssemblyInfo();
+            WriteLogInfo();
             ZipArchive.Dispose();
             ZipArchive = null!;
         }
@@ -259,14 +283,17 @@ void WriteMetadata()
             Metadata.Create(_compilationCount, MetadataVersion).Write(writer);
         }
 
-        void WriteAssemblyInfo()
+        void WriteLogInfo()
         {
-            var entry = ZipArchive.CreateEntry(AssemblyInfoFileName, CompressionLevel.Fastest);
-            using var writer = Polyfill.NewStreamWriter(entry.Open(), ContentEncoding, leaveOpen: false);
-            foreach (var kvp in _mvidToRefInfoMap.OrderBy(x => x.Value.FileName).ThenBy(x => x.Key))
+            var pack = new LogInfoPack()
             {
-                writer.WriteLine($"{kvp.Value.FileName}:{kvp.Key:N}:{kvp.Value.AssemblyName}");
-            }
+                CompilerCallMvidList = _compilerCallMvidList,
+                MvidToReferenceInfoMap = _mvidToRefInfoMap
+            };
+            var contentHash = WriteContentMessagePack(pack);;
+            var entry = ZipArchive.CreateEntry(LogInfoFileName, CompressionLevel.Fastest);
+            using var writer = Polyfill.NewStreamWriter(entry.Open(), ContentEncoding, leaveOpen: false);
+            writer.WriteLine(contentHash);
         }
     }
 
@@ -574,7 +601,7 @@ private void AddAnalyzers(CompilationDataPack dataPack, CommandLineArguments arg
         //
         // Example: .nuget\packages\microsoft.visualstudio.interop\17.2.32505.113\lib\net472\Microsoft.VisualStudio.Interop.dll
         var assemblyName = AssemblyName.GetAssemblyName(filePath);
-        _mvidToRefInfoMap[info.Mvid] = (Path.GetFileName(filePath), assemblyName);
+        _mvidToRefInfoMap[info.Mvid] = (Path.GetFileName(filePath), assemblyName.ToString());
         return info;
     }