refactor: replace Nett with Tomlyn (#230)

* WIP

* Updated some Nett reference -> Toml

* More changes for Nett to Tomlyn

* Changing ref back to Nett since project_assets_2_2 is only used in tests

* Updated Data attributes and removed unnecessary comments

* Updated Data attributes in PoetryLock file

* Made property type more specific

* Formatting fixes

* Made updates to add or ignore rproperties used in Toml Deserialization

* Added documentation for running verification tests

Co-authored-by: Jamie Magee <[email protected]>

Directory.Packages.props

@@ -23,7 +23,6 @@
         <PackageVersion Include="morelinq" Version="3.3.2"/>
         <PackageVersion Include="MSTest.TestAdapter" Version="2.2.10"/>
         <PackageVersion Include="MSTest.TestFramework" Version="2.2.10"/>
-        <PackageVersion Include="Nett" Version="0.15.0"/>
         <PackageVersion Include="Newtonsoft.Json" Version="13.0.1"/>
         <PackageVersion Include="NuGet.ProjectModel" Version="5.11.2"/>
         <PackageVersion Include="NuGet.Versioning" Version="5.11.2"/>
@@ -40,6 +39,7 @@
         <PackageVersion Include="System.Reactive" Version="5.0.0"/>
         <PackageVersion Include="System.Runtime.Loader" Version="4.3.0"/>
         <PackageVersion Include="System.Threading.Tasks.Dataflow" Version="4.9.0"/>
+        <PackageVersion Include="Tomlyn" Version="0.15.0"/>
         <PackageVersion Include="yamldotnet" Version="11.2.1"/>
         <PackageVersion Include="Faker.net" Version="2.0.154"/>
         <PackageVersion Include="Valleysoft.DockerfileModel" Version="1.0.0"/>

docs/running-verification-tests.md

@@ -0,0 +1,42 @@
+# Running Verification Tests On Your Local Machine
+Verification tests are used to confirm that no detectors are lost when changes are made to the project. The tests are run on every PR build. They work by comparing the detection results from the main branch to detection results from the new changes on your PR. You can follow the steps below to run them locally.
+
+## Step 1 : Run Detection on the main branch
+
+- Checkout the main branch in your local repo
+- Create a folder to store detection results (e.g C:\old-output-folder)
+- Use command line to run detection on the main branch with the code below:
+
+```dotnet run scan --Verbosity Verbose --SourceDirectory {path to your local repo} --Output {path to the output folder you created}```
+
+For Example:
+
+```dotnet run scan --Verbosity Verbose --SourceDirectory C:\componentdetection --Output C:\old-output-folder```
+
+
+## Step 2 : Run Detection on your new branch
+
+- Checkout the branch with the new changes you are trying to merge
+- Create a folder to store detection results. This folder should be seperate from the one you used in Step 1 (e.g C:\new-output-folder)
+- Use command line to run detection on the main branch with the code below:
+
+```dotnet run scan --Verbosity Verbose --SourceDirectory {path to your local repo} --Output {path to the output folder you created}```
+
+For Example:
+
+```dotnet run scan --Verbosity Verbose --SourceDirectory C:\componentdetection --Output C:\new-output-folder```
+
+## Step 3 : Update variables in the test
+
+- Open the Microsoft.ComponentDetection.VerificationTests project in VS Studio
+- Navigate to  `GatherResources()`  in `ComponentDetectionIntegrationTests.cs`
+- Update the following variables:
+    -  `oldGithubArtifactsDir` : This should be the output folder you created in Step 1
+    - `newGithubArtifactsDir` : This should be the output folder you created in Step 2
+    - `allowedTimeDriftRatioString`:  This should be ".75"
+
+
+## Step 4: Run The tests
+You can run the tests in two ways:
+- Run or Debug the tests in test explorer.
+- Use the command line to navigate to the Microsoft.ComponentDetection.VerificationTests folder, and run `dotnet test`.

src/Microsoft.ComponentDetection.Detectors/Microsoft.ComponentDetection.Detectors.csproj

@@ -1,9 +1,8 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
     <ItemGroup>
         <PackageReference Include="DotNet.Glob" />
         <PackageReference Include="morelinq" />
-        <PackageReference Include="Nett" />
         <PackageReference Include="NuGet.ProjectModel" />
         <PackageReference Include="NuGet.Versioning" />
         <PackageReference Include="Polly" />
@@ -18,6 +17,7 @@
         <PackageReference Include="System.Composition.TypedParts" />
         <PackageReference Include="System.Reactive" />
         <PackageReference Include="System.Threading.Tasks.Dataflow" />
+        <PackageReference Include="Tomlyn"/>
         <PackageReference Include="Valleysoft.DockerfileModel " />
     </ItemGroup>
 
@@ -34,4 +34,4 @@
         </EmbeddedResource>
     </ItemGroup>
 
-</Project>
+</Project>

src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryLock.cs

@@ -1,11 +1,17 @@
-using System.Diagnostics.CodeAnalysis;
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.Serialization;
 
 namespace Microsoft.ComponentDetection.Detectors.Poetry.Contracts
 {
     // Represents Poetry.Lock file structure.
+    [DataContract]
     public class PoetryLock
     {
-        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
-        public PoetryPackage[] package { get; set; }
+        [DataMember(Name = "Package")]
+        public List<PoetryPackage> Package { get; set; }
+
+        [DataMember(Name = "metadata")]
+        public Dictionary<string, object> Metadata { get; set; }
     }
 }

src/Microsoft.ComponentDetection.Detectors/poetry/Contracts/PoetryPackage.cs

@@ -1,19 +1,27 @@
-using System.Diagnostics.CodeAnalysis;
+using System.Collections.Generic;
+using System.Runtime.Serialization;
 
 namespace Microsoft.ComponentDetection.Detectors.Poetry.Contracts
 {
+    [DataContract]
     public class PoetryPackage
     {
-        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
-        public string category { get; set; }
+        [DataMember(Name = "category")]
+        public string Category { get; set; }
 
-        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
-        public string name { get; set; }
+        [DataMember(Name = "name")]
+        public string Name { get; set; }
 
-        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
-        public string version { get; set; }
+        [DataMember(Name = "version")]
+        public string Version { get; set; }
 
-        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
-        public PoetrySource source { get; set; }
+        [DataMember(Name = "source")]
+        public PoetrySource Source { get; set; }
+
+        [DataMember(Name = "dependencies")]
+        public Dictionary<string, object> Dependencies { get; set; }
+
+        [DataMember(Name = "extras")]
+        public Dictionary<string, object> Extras { get; set; }
     }
 }

src/Microsoft.ComponentDetection.Detectors/poetry/PoetryComponentDetector.cs

@@ -3,11 +3,12 @@
 using System.Composition;
 using System.Linq;
 using System.Threading.Tasks;
+using System.IO;
+using Tomlyn;
 using Microsoft.ComponentDetection.Contracts;
 using Microsoft.ComponentDetection.Contracts.Internal;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
 using Microsoft.ComponentDetection.Detectors.Poetry.Contracts;
-using Nett;
 
 namespace Microsoft.ComponentDetection.Detectors.Poetry
 {
@@ -30,23 +31,27 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary<s
             var poetryLockFile = processRequest.ComponentStream;
             this.Logger.LogVerbose("Found Poetry lockfile: " + poetryLockFile);
 
-            var poetryLock = StreamTomlSerializer.Deserialize(poetryLockFile.Stream, TomlSettings.Create()).Get<PoetryLock>();
-            poetryLock.package.ToList().ForEach(package =>
+            var reader = new StreamReader(poetryLockFile.Stream);
+            var options = new TomlModelOptions
             {
-                var isDevelopmentDependency = package.category != "main";
+                IgnoreMissingProperties = true,
+            };
+            var poetryLock = Toml.ToModel<PoetryLock>(reader.ReadToEnd(), options: options);
+            poetryLock.Package.ToList().ForEach(package =>
+            {
+                var isDevelopmentDependency = package.Category != "main";
 
-                if (package.source != null && package.source.type == "git")
+                if (package.Source != null && package.Source.type == "git")
                 {
-                    var component = new DetectedComponent(new GitComponent(new Uri(package.source.url), package.source.resolved_reference));
+                    var component = new DetectedComponent(new GitComponent(new Uri(package.Source.url), package.Source.resolved_reference));
                     singleFileComponentRecorder.RegisterUsage(component, isDevelopmentDependency: isDevelopmentDependency);
                 }
                 else
                 {
-                    var component = new DetectedComponent(new PipComponent(package.name, package.version));
+                    var component = new DetectedComponent(new PipComponent(package.Name, package.Version));
                     singleFileComponentRecorder.RegisterUsage(component, isDevelopmentDependency: isDevelopmentDependency);
                 }
             });
-
             return Task.CompletedTask;
         }
     }

src/Microsoft.ComponentDetection.Detectors/rust/Contracts/CargoLock.cs

@@ -1,15 +1,19 @@
-using System.Diagnostics.CodeAnalysis;
-using Nett;
+using System.Diagnostics.CodeAnalysis;
 
 namespace Microsoft.ComponentDetection.Detectors.Rust.Contracts
 {
+    using System.Collections.Generic;
+    using System.Runtime.Serialization;
+    using Tomlyn.Model;
+
     // Represents Cargo.Lock file structure.
+    [DataContract]
     public class CargoLock
     {
-        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
-        public CargoPackage[] package { get; set; }
+        [DataMember(Name = "package")]
+        public List<CargoPackage> Package { get; set; }
 
-        [SuppressMessage("StyleCop.CSharp.NamingRules", "SA1300:ElementMustBeginWithUpperCaseLetter", Justification = "Deserialization contract. Casing cannot be overwritten.")]
-        public TomlTable metadata { get; set; }
+        [DataMember(Name = "metadata")]
+        public Dictionary<string, object> Metadata { get; set; }
     }
 }

src/Microsoft.ComponentDetection.Detectors/rust/RustCrateDetector.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Composition;
 using System.IO;
@@ -8,7 +8,7 @@
 using Microsoft.ComponentDetection.Contracts.Internal;
 using Microsoft.ComponentDetection.Contracts.TypedComponent;
 using Microsoft.ComponentDetection.Detectors.Rust.Contracts;
-using Nett;
+using Tomlyn;
 
 namespace Microsoft.ComponentDetection.Detectors.Rust
 {
@@ -32,10 +32,15 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary<s
 
             try
             {
-                var cargoLock = StreamTomlSerializer.Deserialize(cargoLockFile.Stream, TomlSettings.Create()).Get<CargoLock>();
+                var reader = new StreamReader(cargoLockFile.Stream);
+                var options = new TomlModelOptions
+                {
+                    IgnoreMissingProperties = true,
+                };
+                var cargoLock = Toml.ToModel<CargoLock>(reader.ReadToEnd(), options: options);
 
                 // This makes sure we're only trying to parse Cargo.lock v1 formats
-                if (cargoLock.metadata == null)
+                if (cargoLock.Metadata == null)
                 {
                     this.Logger.LogInfo($"Cargo.lock file at {cargoLockFile.Location} contains no metadata section so we're parsing it as the v2 format. The v1 detector will not process it.");
                     return Task.CompletedTask;
@@ -79,7 +84,7 @@ protected override Task OnFileFound(ProcessRequest processRequest, IDictionary<s
                     return Task.CompletedTask;
                 }
 
-                var cargoPackages = cargoLock.package.ToHashSet();
+                var cargoPackages = cargoLock.Package.ToHashSet();
                 RustCrateUtilities.BuildGraph(cargoPackages, cargoDependencyData.NonDevDependencies, cargoDependencyData.DevDependencies, singleFileComponentRecorder);
             }
             catch (Exception e)