Fix issues with EnumerateVersionsAsync not handling 404s (#353)

* Fix all the project managers' EnumerateVersionsAsync to return an empty array if a 404 is hit. Because no versions exist if it 404s. * Add unit tests for the detailed existence methods for NuGet and PyPI. * Fix bug with the NPM detailed package version existence. It only checked if the package existed in the detailed version existence check, it never actually checked to see if the version ever existed. * Override the PrintMembers method in the Removed existence reasons. Now should print the RemovalReasons correctly, vs System.Collections.Generic.HashSet`1[Microsoft.CST.OpenSource.Model.Enums.PackageVersionRemovalReason] * Update NuGet unit tests based on PR comments.
microsoft · Sep 15, 2022 · fb2d403 · fb2d403
1 parent eac404b
commit fb2d403
Show file tree

Hide file tree

Showing 19 changed files with 239 additions and 4 deletions.
diff --git a/src/Shared/Contracts/IPackageExistence.cs b/src/Shared/Contracts/IPackageExistence.cs
@@ -2,9 +2,6 @@
 
 namespace Microsoft.CST.OpenSource.Contracts;
 
-using Model.Enums;
-using System.Collections.Generic;
-
 /// <summary>
 /// An interface representing the existence of a package/version.
 /// </summary>

diff --git a/src/Shared/Model/PackageExistence/PackageExistence.cs b/src/Shared/Model/PackageExistence/PackageExistence.cs
@@ -4,7 +4,10 @@ namespace Microsoft.CST.OpenSource.Model.PackageExistence;
 
 using Contracts;
 using Enums;
+using System;
 using System.Collections.Generic;
+using System.Linq;
+using System.Text;
 
 /// <summary>
 /// Represents a package that currently exists.
@@ -31,4 +34,17 @@ public record PackageNotFound : IPackageExistence
 public record PackageRemoved(IReadOnlySet<PackageRemovalReason> RemovalReasons) : PackageNotFound
 {
     public override bool HasEverExisted => true;
+
+    protected override bool PrintMembers(StringBuilder stringBuilder)
+    {
+        if (base.PrintMembers(stringBuilder))
+        {
+            stringBuilder.Append(", ");
+        }
+
+        string reasons = string.Join(',', this.RemovalReasons.Select(r => r.ToString()));
+
+        stringBuilder.Append($"RemovalReasons = [{reasons}]");
+        return true;
+    }
 }
diff --git a/src/Shared/Model/PackageExistence/PackageVersionExistence.cs b/src/Shared/Model/PackageExistence/PackageVersionExistence.cs
@@ -5,6 +5,8 @@ namespace Microsoft.CST.OpenSource.Model.PackageExistence;
 using Contracts;
 using Enums;
 using System.Collections.Generic;
+using System.Linq;
+using System.Text;
 
 /// <summary>
 /// Represents a package version that currently exists.
@@ -31,4 +33,17 @@ public record PackageVersionNotFound : IPackageExistence
 public record PackageVersionRemoved(IReadOnlySet<PackageVersionRemovalReason> RemovalReasons) : PackageVersionNotFound
 {
     public override bool HasEverExisted => true;
+
+    protected override bool PrintMembers(StringBuilder stringBuilder)
+    {
+        if (base.PrintMembers(stringBuilder))
+        {
+            stringBuilder.Append(", ");
+        }
+
+        string reasons = string.Join(',', this.RemovalReasons.Select(r => r.ToString()));
+
+        stringBuilder.Append($"RemovalReasons = [{reasons}]");
+        return true;
+    }
 }
diff --git a/src/Shared/PackageManagers/CPANProjectManager.cs b/src/Shared/PackageManagers/CPANProjectManager.cs
@@ -9,6 +9,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Text.RegularExpressions;
     using System.Threading.Tasks;
@@ -178,6 +179,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
                 IEnumerable<string> result = SortVersions(versionList.Distinct());
                 return result;
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                return Array.Empty<string>();
+            }
             catch (Exception ex)
             {
                 Logger.Debug("Unable to enumerate versions: {0}", ex.Message);

diff --git a/src/Shared/PackageManagers/CRANProjectManager.cs b/src/Shared/PackageManagers/CRANProjectManager.cs
@@ -9,6 +9,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Threading.Tasks;
 
@@ -169,6 +170,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
                 }
                 return SortVersions(versionList.Distinct());
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                return Array.Empty<string>();
+            }
             catch (Exception ex)
             {
                 Logger.Debug("Unable to enumerate versions: {0}", ex.Message);

diff --git a/src/Shared/PackageManagers/CargoProjectManager.cs b/src/Shared/PackageManagers/CargoProjectManager.cs
@@ -10,6 +10,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Text.Json;
     using System.Threading.Tasks;
@@ -141,6 +142,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
                 }
                 return SortVersions(versionList.Distinct());
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                return Array.Empty<string>();
+            }
             catch (Exception ex)
             {
                 Logger.Debug("Unable to enumerate versions: {0}", ex.Message);

diff --git a/src/Shared/PackageManagers/CocoapodsProjectManager.cs b/src/Shared/PackageManagers/CocoapodsProjectManager.cs
@@ -10,6 +10,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Security.Cryptography;
     using System.Text;
@@ -171,6 +172,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
                 }
                 return SortVersions(versionList.Distinct());
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                return Array.Empty<string>();
+            }
             catch (Exception ex)
             {
                 Logger.Debug("Unable to enumerate versions: {0}", ex.Message);

diff --git a/src/Shared/PackageManagers/ComposerProjectManager.cs b/src/Shared/PackageManagers/ComposerProjectManager.cs
@@ -8,6 +8,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Threading.Tasks;
     using Utilities;
@@ -155,6 +156,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
                 }
                 return SortVersions(versionList.Distinct());
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                return Array.Empty<string>();
+            }
             catch (Exception ex)
             {
                 Logger.Debug("Unable to enumerate versions: {0}", ex.Message);

diff --git a/src/Shared/PackageManagers/GemProjectManager.cs b/src/Shared/PackageManagers/GemProjectManager.cs
@@ -8,6 +8,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Text.Json;
     using System.Text.RegularExpressions;
@@ -134,6 +135,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
                 }
                 return SortVersions(versionList.Distinct());
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                return Array.Empty<string>();
+            }
             catch (Exception ex)
             {
                 Logger.Debug("Unable to enumerate versions: {0}", ex.Message);

diff --git a/src/Shared/PackageManagers/GitHubProjectManager.cs b/src/Shared/PackageManagers/GitHubProjectManager.cs
@@ -9,6 +9,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Diagnostics;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Text.RegularExpressions;
     using System.Threading.Tasks;
@@ -227,6 +228,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
                 }
                 return SortVersions(versionList);
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                return Array.Empty<string>();
+            }
             catch (Exception ex)
             {
                 Logger.Debug("Unable to enumerate versions: {0}", ex.Message);

diff --git a/src/Shared/PackageManagers/GolangProjectManager.cs b/src/Shared/PackageManagers/GolangProjectManager.cs
@@ -8,6 +8,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Threading.Tasks;
 
@@ -140,6 +141,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
                 }
                 return SortVersions(versionList.Distinct());
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                return Array.Empty<string>();
+            }
             catch (Exception ex)
             {
                 Logger.Debug("Unable to enumerate versions: {0}", ex.Message);

diff --git a/src/Shared/PackageManagers/HackageProjectManager.cs b/src/Shared/PackageManagers/HackageProjectManager.cs
@@ -9,6 +9,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Threading.Tasks;
 
@@ -125,6 +126,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
 
                 return SortVersions(versionList.Distinct());
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                return Array.Empty<string>();
+            }
             catch (Exception ex)
             {
                 Logger.Debug("Unable to enumerate versions: {0}", ex.Message);

diff --git a/src/Shared/PackageManagers/MavenProjectManager.cs b/src/Shared/PackageManagers/MavenProjectManager.cs
@@ -8,6 +8,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Threading.Tasks;
     using System.Xml;
@@ -140,6 +141,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
                 }
                 return SortVersions(versionList.Distinct());
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                return Array.Empty<string>();
+            }
             catch (Exception ex)
             {
                 Logger.Debug("Unable to enumerate versions: {0}", ex.Message);

diff --git a/src/Shared/PackageManagers/NPMProjectManager.cs b/src/Shared/PackageManagers/NPMProjectManager.cs
@@ -14,6 +14,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Text.Json;
     using System.Threading.Tasks;
@@ -175,6 +176,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
 
                 return sortedList;
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                return Array.Empty<string>();
+            }
             catch (Exception ex)
             {
                 Logger.Debug("Unable to enumerate versions: {0}", ex.Message);
@@ -538,6 +544,12 @@ public override async Task<IPackageExistence> DetailedPackageVersionExistsAsync(
 
             JsonDocument contentJSON = JsonDocument.Parse(content);
             JsonElement root = contentJSON.RootElement;
+
+            // Check to make sure that the package version ever existed.
+            if (!PackageVersionEverExisted(purl, root))
+            {
+                return new PackageVersionNotFound();
+            }
 
             HashSet<PackageVersionRemovalReason> removalReasons = new();
 
@@ -616,7 +628,7 @@ internal virtual bool PackageVersionPulled(PackageURL purl, JsonElement root)
         }
 
         /// <summary>
-        /// Check to see if the package only has a NPM security holding package.
+        /// Check to see if the package has a NPM security holding package.
         /// </summary>
         /// <param name="root">The <see cref="JsonElement"/> root content of the metadata.</param>
         /// <returns>True if this package is a NPM security holding package. False otherwise.</returns>
@@ -626,6 +638,35 @@ internal virtual bool PackageConsideredMalicious(JsonElement root)
             return time.EnumerateObject().Any(timeEntry => timeEntry.Name == NPM_SECURITY_HOLDING_VERSION);
         }
 
+        /// <summary>
+        /// Check to see if the package version ever existed.
+        /// </summary>
+        /// <param name="purl">The <see cref="PackageURL"/> to check.</param>
+        /// <param name="root">The <see cref="JsonElement"/> root content of the metadata.</param>
+        /// <returns>True if this package version ever existed. False otherwise.</returns>
+        internal virtual bool PackageVersionEverExisted(PackageURL purl, JsonElement root)
+        {
+            // Did this version ever exist? Start with assuming not.
+            bool everExisted = false;
+
+            JsonElement time = root.GetProperty("time");
+
+            // Check the unpublished property in time if it exists for the purl's version.
+            if (time.TryGetProperty("unpublished", out JsonElement unpublishedElement))
+            {
+                List<string>? versions = OssUtilities.ConvertJSONToList(OssUtilities.GetJSONPropertyIfExists(unpublishedElement, "versions"));
+                everExisted = versions?.Contains(purl.Version) ?? false;
+            }
+
+            // If the version wasn't in unpublished, then check if any of the versions in time match.
+            if (!everExisted)
+            {
+                everExisted = time.EnumerateObject().Any(timeEntry => timeEntry.Name == purl.Version);
+            }
+
+            return everExisted;
+        }
+
         /// <summary>
         /// Searches the package manager metadata to figure out the source code repository
         /// </summary>

diff --git a/src/Shared/PackageManagers/PyPIProjectManager.cs b/src/Shared/PackageManagers/PyPIProjectManager.cs
@@ -12,6 +12,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Text.Json;
     using System.Threading.Tasks;
@@ -180,6 +181,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
 
                 return SortVersions(versionList.Distinct());
             }
+            catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+            {
+                Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                return Array.Empty<string>();
+            }
             catch (Exception ex)
             {
                 Logger.Debug("Unable to enumerate versions: {0}", ex.Message);

diff --git a/src/Shared/PackageManagers/UbuntuProjectManager.cs b/src/Shared/PackageManagers/UbuntuProjectManager.cs
@@ -9,6 +9,7 @@ namespace Microsoft.CST.OpenSource.PackageManagers
     using System.Collections.Generic;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Net.Http;
     using System.Text;
     using System.Text.RegularExpressions;
@@ -245,6 +246,11 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
                         }
                     }
                 }
+                catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
+                {
+                    Logger.Debug("Unable to enumerate versions (404): {0}", ex.Message);
+                    return Array.Empty<string>();
+                }
                 catch (Exception ex)
                 {
                     Logger.Debug(ex, "Error enumerating Ubuntu package versions: {0}", ex.Message);