Solution based merging for data collector

Documentation/Changelog.md

@@ -6,7 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
-## Release date 2024-01-20
+### Improvements
+- Implement solution based merging for data collector [#1307](https://github.com/coverlet-coverage/coverlet/issues/1307)
+
+## Release date 2025-01-20
 ### Packages
 coverlet.msbuild 6.0.4
 coverlet.console 6.0.4

Documentation/Troubleshooting.md

@@ -265,3 +265,12 @@ To enable exceptions log for in-process data collectors
 ```shell
  set COVERLET_DATACOLLECTOR_INPROC_EXCEPTIONLOG_ENABLED=1
 ```
+
+## Enable collector post processing debugging
+Post processing is uses for automatically merging coverage reports with the `ReportMerging` option enabled. You can live attach and debug the post processor `COVERLET_DATACOLLECTOR_POSTPROCESSOR_DEBUG`.
+
+You will be asked to attach a debugger through UI popup.
+
+```shell
+ set COVERLET_DATACOLLECTOR_POSTPROCESSOR_DEBUG=1
+```

Documentation/VSTestIntegration.md

@@ -121,6 +121,7 @@ These are a list of options that are supported by coverlet. These can be specifi
 | DoesNotReturnAttribute   | Methods marked with these attributes are known not to return, statements following them will be excluded from coverage                                          |
 | DeterministicReport      | Generates deterministic report in context of deterministic build. Take a look at [documentation](DeterministicBuild.md) for further informations.
 | ExcludeAssembliesWithoutSources |  Specifies whether to exclude assemblies without source. Options are either MissingAll, MissingAny or None. Default is MissingAll.|
+| ReportMerging | Automatically merge coverage reports if coverage is calculated for multiple projects. Default is false.|
 
 How to specify these options via runsettings?
 
@@ -143,6 +144,7 @@ How to specify these options via runsettings?
           <SkipAutoProps>true</SkipAutoProps>
           <DeterministicReport>false</DeterministicReport>
           <ExcludeAssembliesWithoutSources>MissingAll,MissingAny,None</ExcludeAssembliesWithoutSources>
+          <ReportMerging>false</ReportMerging>
         </Configuration>
       </DataCollector>
     </DataCollectors>

src/coverlet.collector/ArtifactPostProcessor/CoverletCoveragePostProcessor.cs

@@ -0,0 +1,162 @@
+// Copyright (c) Toni Solarin-Sodara
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using System.Xml;
+using Coverlet.Collector.Utilities;
+using Coverlet.Core;
+using Coverlet.Core.Abstractions;
+using Coverlet.Core.Reporters;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.DataCollection;
+using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
+using Newtonsoft.Json;
+using System.Diagnostics;
+using Coverlet.Core.Helpers;
+
+namespace coverlet.collector.ArtifactPostProcessor
+{
+  public class CoverletCoveragePostProcessor : IDataCollectorAttachmentProcessor
+  {
+    private CoverageResult _coverageResult;
+    private ReportFormatParser _reportFormatParser;
+    private IMessageLogger _logger;
+
+    public bool SupportsIncrementalProcessing => true;
+
+    public IEnumerable<Uri> GetExtensionUris() => new[] { new Uri(CoverletConstants.DefaultUri) };
+
+    public Task<ICollection<AttachmentSet>> ProcessAttachmentSetsAsync(XmlElement configurationElement,
+      ICollection<AttachmentSet> attachments, IProgress<int> progressReporter,
+      IMessageLogger logger, CancellationToken cancellationToken)
+    {
+      _reportFormatParser ??= new ReportFormatParser();
+      _coverageResult ??= new CoverageResult();
+      _coverageResult.Modules ??= new Modules();
+      _logger = logger;
+
+      string[] formats = _reportFormatParser.ParseReportFormats(configurationElement);
+      bool deterministic = _reportFormatParser.ParseDeterministicReport(configurationElement);
+      bool useSourceLink = _reportFormatParser.ParseUseSourceLink(configurationElement);
+      bool reportMerging = _reportFormatParser.ParseReportMerging(configurationElement);
+
+      AttachDebugger();
+
+      if (!reportMerging) return Task.FromResult(attachments);
+
+      IList<IReporter> reporters = CreateReporters(formats).ToList();
+
+      if (attachments.Count > 1)
+      {
+        _coverageResult.Parameters = new CoverageParameters() {DeterministicReport = deterministic, UseSourceLink = useSourceLink };
+
+        var fileAttachments = attachments.SelectMany(x => x.Attachments.Where(IsFileAttachment)).ToList();
+        string mergeFilePath = Path.GetDirectoryName(fileAttachments.First().Uri.LocalPath);
+
+        MergeExistingJsonReports(attachments);
+
+        RemoveObsoleteReports(fileAttachments);
+
+        AttachmentSet mergedFileAttachment = WriteCoverageReports(reporters, mergeFilePath, _coverageResult);
+
+        attachments = new List<AttachmentSet> { mergedFileAttachment };
+      }
+
+      return Task.FromResult(attachments);
+    }
+
+    private static void RemoveObsoleteReports(List<UriDataAttachment> fileAttachments)
+    {
+      fileAttachments.ForEach(x =>
+      {
+        string directory = Path.GetDirectoryName(x.Uri.LocalPath);
+        if (! string.IsNullOrEmpty(directory) && Directory.Exists(directory))
+          Directory.Delete(directory, true);
+      });
+    }
+
+    private void MergeExistingJsonReports(IEnumerable<AttachmentSet> attachments)
+    {
+      foreach (AttachmentSet attachmentSet in attachments)
+      {
+        attachmentSet.Attachments.Where(IsFileWithJsonExt).ToList().ForEach(x =>
+          MergeWithCoverageResult(x.Uri.LocalPath, _coverageResult)
+        );
+      }
+    }
+
+    private AttachmentSet WriteCoverageReports(IEnumerable<IReporter> reporters, string directory, CoverageResult coverageResult)
+    {
+      var attachment = new AttachmentSet(new Uri(CoverletConstants.DefaultUri), string.Empty);
+      foreach (IReporter reporter in reporters)
+      {
+        string report = GetCoverageReport(coverageResult, reporter);
+        var file = new FileInfo(Path.Combine(directory, Path.ChangeExtension(CoverletConstants.DefaultFileName, reporter.Extension)));
+        file.Directory?.Create();
+        File.WriteAllText(file.FullName, report);
+        attachment.Attachments.Add(new UriDataAttachment(new Uri(file.FullName),string.Empty));
+      }
+      return attachment;
+    }
+
+    private static bool IsFileWithJsonExt(UriDataAttachment x)
+    {
+      return IsFileAttachment(x) && Path.GetExtension(x.Uri.AbsolutePath).Equals(".json");
+    }
+
+    private static bool IsFileAttachment(UriDataAttachment x)
+    {
+      return x.Uri.IsFile;
+    }
+
+    private void MergeWithCoverageResult(string filePath, CoverageResult coverageResult)
+    {
+      string json = File.ReadAllText(filePath);
+      coverageResult.Merge(JsonConvert.DeserializeObject<Modules>(json));
+    }
+
+    private string GetCoverageReport(CoverageResult coverageResult, IReporter reporter)
+    {
+      try
+      {
+        // empty source root translator returns the original path for deterministic report
+        return reporter.Report(coverageResult, new SourceRootTranslator());
+      }
+      catch (Exception ex)
+      {
+        throw new CoverletDataCollectorException(
+          $"{CoverletConstants.DataCollectorName}: Failed to get coverage report", ex);
+      }
+    }
+
+    private void AttachDebugger()
+    {
+      if (int.TryParse(Environment.GetEnvironmentVariable("COVERLET_DATACOLLECTOR_POSTPROCESSOR_DEBUG"), out int result) && result == 1)
+      {
+        Debugger.Launch();
+        Debugger.Break();
+      }
+    }
+
+    private IEnumerable<IReporter> CreateReporters(IEnumerable<string> formats)
+    {
+      IEnumerable<IReporter> reporters = formats.Select(format =>
+      {
+        var reporterFactory = new ReporterFactory(format);
+        if (!reporterFactory.IsValidFormat())
+        {
+          _logger.SendMessage(TestMessageLevel.Warning, $"Invalid report format '{format}'");
+          return null;
+        }
+        return reporterFactory.CreateReporter();
+      }).Where(r => r != null);
+
+      return reporters;
+    }
+  }
+}

src/coverlet.collector/DataCollection/CoverageManager.cs

@@ -26,24 +26,32 @@ internal class CoverageManager
     public CoverageManager(CoverletSettings settings, TestPlatformEqtTrace eqtTrace, TestPlatformLogger logger, ICoverageWrapper coverageWrapper,
                            IInstrumentationHelper instrumentationHelper, IFileSystem fileSystem, ISourceRootTranslator sourceRootTranslator, ICecilSymbolHelper cecilSymbolHelper)
         : this(settings,
-        settings.ReportFormats.Select(format =>
-        {
-          var reporterFactory = new ReporterFactory(format);
-          if (!reporterFactory.IsValidFormat())
-          {
-            eqtTrace.Warning($"Invalid report format '{format}'");
-            return null;
-          }
-          else
-          {
-            return reporterFactory.CreateReporter();
-          }
-        }).Where(r => r != null).ToArray(),
+        CreateReporters(settings, eqtTrace),
         new CoverletLogger(eqtTrace, logger),
         coverageWrapper, instrumentationHelper, fileSystem, sourceRootTranslator, cecilSymbolHelper)
     {
     }
 
+    private static IReporter[] CreateReporters(CoverletSettings settings, TestPlatformEqtTrace eqtTrace)
+    {
+      if (settings.ReportMerging && ! settings.ReportFormats.Contains("json"))
+        settings.ReportFormats = settings.ReportFormats.Append("json").ToArray();
+
+      return settings.ReportFormats.Select(format =>
+      {
+        var reporterFactory = new ReporterFactory(format);
+        if (!reporterFactory.IsValidFormat())
+        {
+          eqtTrace.Warning($"Invalid report format '{format}'");
+          return null;
+        }
+        else
+        {
+          return reporterFactory.CreateReporter();
+        }
+      }).Where(r => r != null).ToArray();
+    }
+
     public CoverageManager(CoverletSettings settings, IReporter[] reporters, ILogger logger, ICoverageWrapper coverageWrapper,
                            IInstrumentationHelper instrumentationHelper, IFileSystem fileSystem, ISourceRootTranslator sourceRootTranslator, ICecilSymbolHelper cecilSymbolHelper)
     {

src/coverlet.collector/DataCollection/CoverletCoverageCollector.cs

@@ -6,6 +6,7 @@
 using System.Diagnostics;
 using System.Linq;
 using System.Xml;
+using coverlet.collector.ArtifactPostProcessor;
 using Coverlet.Collector.Utilities;
 using Coverlet.Collector.Utilities.Interfaces;
 using Coverlet.Core.Abstractions;
@@ -21,6 +22,7 @@ namespace Coverlet.Collector.DataCollection
   /// </summary>
   [DataCollectorTypeUri(CoverletConstants.DefaultUri)]
   [DataCollectorFriendlyName(CoverletConstants.FriendlyName)]
+  [DataCollectorAttachmentProcessor(typeof(CoverletCoveragePostProcessor))]
   public class CoverletCoverageCollector : DataCollector
   {
     private readonly TestPlatformEqtTrace _eqtTrace;

src/coverlet.collector/DataCollection/CoverletSettings.cs

@@ -86,6 +86,11 @@ internal class CoverletSettings
     /// </summary>
     public string ExcludeAssembliesWithoutSources { get; set; }
 
+    /// <summary>
+    /// Report merging flag
+    /// </summary>
+    public bool ReportMerging { get; set; }
+
     public override string ToString()
     {
       var builder = new StringBuilder();
@@ -104,6 +109,7 @@ public override string ToString()
       builder.AppendFormat("DoesNotReturnAttributes: '{0}'", string.Join(",", DoesNotReturnAttributes ?? Enumerable.Empty<string>()));
       builder.AppendFormat("DeterministicReport: '{0}'", DeterministicReport);
       builder.AppendFormat("ExcludeAssembliesWithoutSources: '{0}'", ExcludeAssembliesWithoutSources);
+      builder.AppendFormat("ReportMerging: '{0}'", ReportMerging);
 
       return builder.ToString();
     }

src/coverlet.collector/DataCollection/CoverletSettingsParser.cs

@@ -15,10 +15,12 @@ namespace Coverlet.Collector.DataCollection
   internal class CoverletSettingsParser
   {
     private readonly TestPlatformEqtTrace _eqtTrace;
+    private readonly ReportFormatParser _reportFormatParser;
 
     public CoverletSettingsParser(TestPlatformEqtTrace eqtTrace)
     {
       _eqtTrace = eqtTrace;
+      _reportFormatParser = new ReportFormatParser();
     }
 
     /// <summary>
@@ -48,9 +50,10 @@ public CoverletSettings Parse(XmlElement configurationElement, IEnumerable<strin
         coverletSettings.DoesNotReturnAttributes = ParseDoesNotReturnAttributes(configurationElement);
         coverletSettings.DeterministicReport = ParseDeterministicReport(configurationElement);
         coverletSettings.ExcludeAssembliesWithoutSources = ParseExcludeAssembliesWithoutSources(configurationElement);
+        coverletSettings.ReportMerging = _reportFormatParser.ParseReportMerging(configurationElement);
       }
 
-      coverletSettings.ReportFormats = ParseReportFormats(configurationElement);
+      coverletSettings.ReportFormats = _reportFormatParser.ParseReportFormats(configurationElement);
       coverletSettings.ExcludeFilters = ParseExcludeFilters(configurationElement);
 
       if (_eqtTrace.IsVerboseEnabled)
@@ -80,23 +83,6 @@ private static string ParseTestModule(IEnumerable<string> testModules)
       return testModules.FirstOrDefault();
     }
 
-    /// <summary>
-    /// Parse report formats
-    /// </summary>
-    /// <param name="configurationElement">Configuration element</param>
-    /// <returns>Report formats</returns>
-    private static string[] ParseReportFormats(XmlElement configurationElement)
-    {
-      string[] formats = Array.Empty<string>();
-      if (configurationElement != null)
-      {
-        XmlElement reportFormatElement = configurationElement[CoverletConstants.ReportFormatElementName];
-        formats = SplitElement(reportFormatElement);
-      }
-
-      return formats is null || formats.Length == 0 ? new[] { CoverletConstants.DefaultReportFormat } : formats;
-    }
-
     /// <summary>
     /// Parse filters to include
     /// </summary>

src/coverlet.collector/Utilities/CoverletConstants.cs

@@ -27,5 +27,6 @@ internal static class CoverletConstants
     public const string DoesNotReturnAttributesElementName = "DoesNotReturnAttribute";
     public const string DeterministicReport = "DeterministicReport";
     public const string ExcludeAssembliesWithoutSources = "ExcludeAssembliesWithoutSources";
+    public const string ReportMerging = "ReportMerging";
   }
 }

src/coverlet.collector/Utilities/ReportFormatParser.cs

@@ -0,0 +1,50 @@
+// Copyright (c) Toni Solarin-Sodara
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Xml;
+using System.Linq;
+
+namespace Coverlet.Collector.Utilities
+{
+  internal class ReportFormatParser
+  {
+    internal string[] ParseReportFormats(XmlElement configurationElement)
+    {
+      string[] formats = Array.Empty<string>();
+      if (configurationElement != null)
+      {
+        XmlElement reportFormatElement = configurationElement[CoverletConstants.ReportFormatElementName];
+        formats = SplitElement(reportFormatElement);
+      }
+
+      return formats is null || formats.Length == 0 ? new[] { CoverletConstants.DefaultReportFormat } : formats;
+    }
+
+    private static string[] SplitElement(XmlElement element)
+    {
+      return element?.InnerText?.Split(new char[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Where(value => !string.IsNullOrWhiteSpace(value)).Select(value => value.Trim()).ToArray();
+    }
+
+    internal bool ParseUseSourceLink(XmlElement configurationElement)
+    {
+      XmlElement useSourceLinkElement = configurationElement[CoverletConstants.UseSourceLinkElementName];
+      bool.TryParse(useSourceLinkElement?.InnerText, out bool useSourceLink);
+      return useSourceLink;
+    }
+
+    internal bool ParseDeterministicReport(XmlElement configurationElement)
+    {
+      XmlElement deterministicReportElement = configurationElement[CoverletConstants.DeterministicReport];
+      bool.TryParse(deterministicReportElement?.InnerText, out bool deterministicReport);
+      return deterministicReport;
+    }
+
+    internal bool ParseReportMerging(XmlElement configurationElement)
+    {
+      XmlElement mergeWithElement = configurationElement[CoverletConstants.ReportMerging];
+      bool.TryParse(mergeWithElement?.InnerText, out bool mergeWith);
+      return mergeWith;
+    }
+  }
+}