Pauldorsch/pipreport version fix (#1229)

* check for valid python versions before adding to the dependency graph

* bump version

* compiled regex

src/Microsoft.ComponentDetection.Detectors/pip/PipReportComponentDetector.cs

@@ -77,7 +77,7 @@ private enum PipReportOverrideBehavior
 
     public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = new[] { ComponentType.Pip };
 
-    public override int Version { get; } = 7;
+    public override int Version { get; } = 8;
 
     protected override bool EnableParallelism { get; set; } = true;
 
@@ -332,6 +332,16 @@ private Dictionary<string, PipReportGraphNode> BuildGraphFromInstallationReport(
                 continue;
             }
 
+            if (!PipReportUtilities.IsCanonicalVersion(package.Metadata.Version))
+            {
+                this.Logger.LogWarning(
+                    "PipReport: Skipping package '{Package}' with non-canonical version '{Version}'. " +
+                    "See https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions",
+                    normalizedPkgName,
+                    package.Metadata.Version);
+                continue;
+            }
+
             var node = new PipReportGraphNode(
                 new PipComponent(
                     normalizedPkgName,

src/Microsoft.ComponentDetection.Detectors/pip/PipReportUtilities.cs

@@ -9,6 +9,12 @@ internal class PipReportUtilities
     private const string ClassifierFieldSeparator = " :: ";
     private const string ClassifierFieldLicensePrefix = "License";
 
+    // Python regular expression for version schema:
+    // https://www.python.org/dev/peps/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions
+    public static readonly Regex CanonicalVersionPatternMatch = new Regex(
+        @"^([1-9]\d*!)?(0|[1-9]\d*)(\.(0|[1-9]\d*))*((a|b|rc)(0|[1-9]\d*))?(\.post(0|[1-9]\d*))?(\.dev(0|[1-9]\d*))?(\+(?:(?<local>[a-z0-9]+(?:[.][a-z0-9]+)*))?)?$",
+        RegexOptions.Compiled);
+
     /// <summary>
     /// Normalize the package name format to the standard Python Packaging format.
     /// See https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization.
@@ -68,4 +74,9 @@ public static string GetLicenseFromInstalledItem(PipInstallationReportItem compo
 
         return null;
     }
+
+    public static bool IsCanonicalVersion(string version)
+    {
+        return CanonicalVersionPatternMatch.Match(version).Success;
+    }
 }

test/Microsoft.ComponentDetection.Detectors.Tests/Microsoft.ComponentDetection.Detectors.Tests.csproj

@@ -40,6 +40,9 @@
         <None Update="Mocks\pip_report_jupyterlab.json">
           <CopyToOutputDirectory>Always</CopyToOutputDirectory>
         </None>
+        <None Update="Mocks\pip_report_single_pkg_invalid_pkg_version.json">
+          <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+        </None>
         <None Update="Mocks\pip_report_single_pkg.json">
             <CopyToOutputDirectory>Always</CopyToOutputDirectory>
         </None>

test/Microsoft.ComponentDetection.Detectors.Tests/Mocks/TestResources.resx

@@ -139,6 +139,9 @@
   <data name="pip_report_single_pkg_bad_version" type="System.Resources.ResXFileRef, System.Windows.Forms">
     <value>pip_report_single_pkg_bad_version.json;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
   </data>
+  <data name="pip_report_single_pkg_invalid_pkg_version" type="System.Resources.ResXFileRef, System.Windows.Forms">
+    <value>pip_report_single_pkg_invalid_pkg_version.json;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
+  </data>
   <data name="project_assets_2_2" type="System.Resources.ResXFileRef, System.Windows.Forms">
     <value>..\Resources\project_assets_2_2.json;System.String, mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089;Windows-1252</value>
   </data>

test/Microsoft.ComponentDetection.Detectors.Tests/Mocks/pip_report_single_pkg_invalid_pkg_version.json

@@ -0,0 +1,57 @@
+{
+  "version": "1",
+  "pip_version": "24.0",
+  "install": [
+    {
+      "download_info": {
+        "url": "https://files.pythonhosted.org/packages/d9/5a/e7c31adbe875f2abbb91bd84cf2dc52d792b5a01506781dbcf25c91daf11/six-1.16.0-py2.py3-none-any.whl",
+        "archive_info": {
+          "hash": "sha256=8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254",
+          "hashes": {
+            "sha256": "8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+          }
+        }
+      },
+      "is_direct": false,
+      "is_yanked": false,
+      "requested": true,
+      "metadata": {
+        "metadata_version": "2.1",
+        "name": "six",
+        "version": ".1",
+        "platform": [
+          "UNKNOWN"
+        ],
+        "summary": "Python 2 and 3 compatibility utilities",
+        "description": ".. image:: https://img.shields.io/pypi/v/six.svg\n   :target: https://pypi.org/project/six/\n   :alt: six on PyPI\n\n.. image:: https://travis-ci.org/benjaminp/six.svg?branch=master\n   :target: https://travis-ci.org/benjaminp/six\n   :alt: six on TravisCI\n\n.. image:: https://readthedocs.org/projects/six/badge/?version=latest\n   :target: https://six.readthedocs.io/\n   :alt: six's documentation on Read the Docs\n\n.. image:: https://img.shields.io/badge/license-MIT-green.svg\n   :target: https://github.com/benjaminp/six/blob/master/LICENSE\n   :alt: MIT License badge\n\nSix is a Python 2 and 3 compatibility library.  It provides utility functions\nfor smoothing over the differences between the Python versions with the goal of\nwriting Python code that is compatible on both Python versions.  See the\ndocumentation for more information on what is provided.\n\nSix supports Python 2.7 and 3.3+.  It is contained in only one Python\nfile, so it can be easily copied into your project. (The copyright and license\nnotice must be retained.)\n\nOnline documentation is at https://six.readthedocs.io/.\n\nBugs can be reported to https://github.com/benjaminp/six.  The code can also\nbe found there.\n\n\n",
+        "home_page": "https://github.com/benjaminp/six",
+        "author": "Benjamin Peterson",
+        "author_email": "[email protected]",
+        "license": "MIT",
+        "classifier": [
+          "Development Status :: 5 - Production/Stable",
+          "Programming Language :: Python :: 2",
+          "Programming Language :: Python :: 3",
+          "Intended Audience :: Developers",
+          "License :: OSI Approved :: MIT License",
+          "Topic :: Software Development :: Libraries",
+          "Topic :: Utilities"
+        ],
+        "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
+      }
+    }
+  ],
+  "environment": {
+    "implementation_name": "cpython",
+    "implementation_version": "3.12.3",
+    "os_name": "nt",
+    "platform_machine": "AMD64",
+    "platform_release": "11",
+    "platform_system": "Windows",
+    "platform_version": "10.0.22631",
+    "python_full_version": "3.12.3",
+    "platform_python_implementation": "CPython",
+    "python_version": "3.12",
+    "sys_platform": "win32"
+  }
+}

test/Microsoft.ComponentDetection.Detectors.Tests/PipReportComponentDetectorTests.cs

@@ -32,6 +32,7 @@ public class PipReportComponentDetectorTests : BaseDetectorTest<PipReportCompone
 
     private readonly PipInstallationReport singlePackageReport;
     private readonly PipInstallationReport singlePackageReportBadVersion;
+    private readonly PipInstallationReport singlePackageReportInvalidPkgVersion;
     private readonly PipInstallationReport multiPackageReport;
     private readonly PipInstallationReport jupyterPackageReport;
     private readonly PipInstallationReport simpleExtrasReport;
@@ -64,6 +65,7 @@ public PipReportComponentDetectorTests()
 
         this.singlePackageReport = JsonConvert.DeserializeObject<PipInstallationReport>(TestResources.pip_report_single_pkg);
         this.singlePackageReportBadVersion = JsonConvert.DeserializeObject<PipInstallationReport>(TestResources.pip_report_single_pkg_bad_version);
+        this.singlePackageReportInvalidPkgVersion = JsonConvert.DeserializeObject<PipInstallationReport>(TestResources.pip_report_single_pkg_invalid_pkg_version);
         this.multiPackageReport = JsonConvert.DeserializeObject<PipInstallationReport>(TestResources.pip_report_multi_pkg);
         this.jupyterPackageReport = JsonConvert.DeserializeObject<PipInstallationReport>(TestResources.pip_report_jupyterlab);
         this.simpleExtrasReport = JsonConvert.DeserializeObject<PipInstallationReport>(TestResources.pip_report_simple_extras);
@@ -256,6 +258,29 @@ public async Task TestPipReportDetector_SimpleExtrasAsync()
         requestsComponent.Version.Should().Be("2.32.3");
     }
 
+    [TestMethod]
+    public async Task TestPipReportDetector_BadPackageVersionAsync()
+    {
+        this.pipCommandService.Setup(x => x.GenerateInstallationReportAsync(It.IsAny<string>(), It.IsAny<string>(), It.IsAny<string>(), It.IsAny<CancellationToken>()))
+            .ReturnsAsync((this.singlePackageReportInvalidPkgVersion, null));
+
+        var (result, componentRecorder) = await this.DetectorTestUtility
+            .WithFile("requirements.txt", string.Empty)
+            .ExecuteDetectorAsync();
+
+        result.ResultCode.Should().Be(ProcessingResultCode.Success);
+
+        this.mockLogger.Verify(x => x.Log(
+            LogLevel.Warning,
+            It.IsAny<EventId>(),
+            It.Is<It.IsAnyType>((o, t) => o.ToString().Contains("with non-canonical version")),
+            It.IsAny<Exception>(),
+            (Func<It.IsAnyType, Exception, string>)It.IsAny<object>()));
+
+        var detectedComponents = componentRecorder.GetDetectedComponents();
+        detectedComponents.Should().BeEmpty();
+    }
+
     [TestMethod]
     public async Task TestPipReportDetector_MultiComponentAsync()
     {

test/Microsoft.ComponentDetection.Detectors.Tests/PipReportUtilitiesTests.cs

@@ -28,4 +28,34 @@ public void NormalizePackageName_ExpectedEquivalent()
         PipReportUtilities.NormalizePackageNameFormat("friendly--bard").Should().Be(normalizedForm);
         PipReportUtilities.NormalizePackageNameFormat("FrIeNdLy-._.-bArD").Should().Be(normalizedForm);
     }
+
+    [TestMethod]
+    [DataRow("1.0.0")]
+    [DataRow("2012.10")]
+    [DataRow("0.1rc2.post3")]
+    [DataRow("1.0.post2.dev3")]
+    [DataRow("1!2.0")]
+    [DataRow("1.0.post1")]
+    public void IsCanonicalVersion_ReturnsTrue_WhenVersionMatchesCanonicalVersionPattern(string version)
+    {
+        // Act
+        var isCanonicalVersion = PipReportUtilities.IsCanonicalVersion(version);
+
+        // Assert
+        isCanonicalVersion.Should().BeTrue();
+    }
+
+    [TestMethod]
+    [DataRow("0.0.1-beta")]
+    [DataRow("1.0.0-alpha.1")]
+    [DataRow(".1")]
+    [DataRow("-pkg-version-")]
+    public void IsCanonicalVersion_ReturnsFalse_WhenVersionDoesNotMatchCanonicalVersionPattern(string version)
+    {
+        // Act
+        var isCanonicalVersion = PipReportUtilities.IsCanonicalVersion(version);
+
+        // Assert
+        isCanonicalVersion.Should().BeFalse();
+    }
 }