Add method to NpmPackageManager to fime the time at which the first p…

…ackage version is created. (#460)

* Add method to fetch when the package is first published.

* Address PR comments.

* Include repo metadata if set.

* Added new argument to GetPackageMetadata base method.

---------

Co-authored-by: Mounika Rendedla <[email protected]>

src/Shared/Contracts/IBaseProjectManager.cs

@@ -113,9 +113,10 @@ public interface IBaseProjectManager
     /// <param name="purl">The <see cref="PackageURL"/> to get the normalized metadata for.</param>
     /// <param name="includePrerelease">If pre-releases should count for getting the latest version, and the list of versions. Defaults to <c>false</c>.</param>
     /// <param name="useCache">If the <see cref="PackageMetadata"/> should be retrieved from the cache, if it is available.</param>
+    /// <param name="includeRepositoryMetadata"> If repository metadata should be retrieved or not. </param>
     /// <remarks>If no version specified, defaults to latest version.</remarks>
     /// <returns>A <see cref="PackageMetadata"/> object representing this <see cref="PackageURL"/>.</returns>
-    public Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true);
+    public Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true, bool includeRepositoryMetadata = true);
 
     /// <summary>
     /// Gets everything contained in a JSON element for the package version

src/Shared/Model/PackageMetadata.cs

@@ -114,6 +114,9 @@ public class PackageMetadata
         [JsonProperty(PropertyName = "upload_time", NullValueHandling = NullValueHandling.Ignore)]
         public DateTime? UploadTime { get; set; }
 
+        [JsonProperty(PropertyName = "created_time", NullValueHandling = NullValueHandling.Ignore)]
+        public DateTime? CreatedTime { get; set; }
+
         [JsonProperty(PropertyName = "commit_id", NullValueHandling = NullValueHandling.Ignore)]
         public string? CommitId { get; set; }
 

src/Shared/PackageManagers/BaseProjectManager.cs

@@ -391,7 +391,7 @@ public virtual async Task<IPackageExistence> DetailedPackageVersionExistsAsync(P
         }
 
         /// <inheritdoc />
-        public virtual Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true)
+        public virtual Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true, bool includeRepositoryMetadata = true)
         {
             string typeName = GetType().Name;
             throw new NotImplementedException($"{typeName} does not implement GetPackageMetadata.");

src/Shared/PackageManagers/CargoProjectManager.cs

@@ -194,7 +194,7 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
             }
         }
 
-        public override async Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true)
+        public override async Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true, bool includeRepositoryMetadata = true)
         {
             string? content = await GetMetadataAsync(purl, useCache);
             if (string.IsNullOrEmpty(content)) { return null; }

src/Shared/PackageManagers/GolangProjectManager.cs

@@ -216,7 +216,7 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
             }
         }
 
-        public override async Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true)
+        public override async Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true, bool includeRepositoryMetadata = true)
         {
             string? content = await GetMetadataAsync(purl, useCache);
             if (string.IsNullOrEmpty(content)) { return null; }

src/Shared/PackageManagers/MavenProjectManager.cs

@@ -267,7 +267,7 @@ public override async Task<bool> PackageVersionExistsAsync(PackageURL purl, bool
             }
         }
 
-        public override async Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true)
+        public override async Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true, bool includeRepositoryMetadata = true)
         {
             string? content = await GetMetadataAsync(purl, useCache);
             if (string.IsNullOrEmpty(content)) { return null; }

src/Shared/PackageManagers/NPMProjectManager.cs

@@ -277,7 +277,7 @@ public override Uri GetPackageAbsoluteUri(PackageURL purl)
 
         /// <inheritdoc />
         /// <remarks>Currently doesn't respect the <paramref name="includePrerelease"/> flag.</remarks>
-        public override async Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true)
+        public override async Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true, bool includeRepositoryMetadata = true)
         {
             PackageMetadata metadata = new();
             string? content = await GetMetadataAsync(purl, useCache);
@@ -295,6 +295,7 @@ public override Uri GetPackageAbsoluteUri(PackageURL purl)
             metadata.Language = "JavaScript";
             metadata.PackageUri = $"{metadata.PackageManagerUri}/package/{metadata.Name}";
             metadata.ApiPackageUri = $"{ENV_NPM_API_ENDPOINT}/{metadata.Name}";
+            metadata.CreatedTime = ParseCreatedTime(contentJSON);
 
             List<Version> versions = GetVersions(contentJSON);
             Version? latestVersion = GetLatestVersion(versions);
@@ -328,7 +329,7 @@ public override Uri GetPackageAbsoluteUri(PackageURL purl)
                     {
                         metadata.Description = description;
                     }
-                    
+
                     JsonElement? distElement = OssUtilities.GetJSONPropertyIfExists(versionElement, "dist");
                     if (OssUtilities.GetJSONPropertyIfExists(distElement, "tarball") is JsonElement tarballElement)
                     {
@@ -349,7 +350,7 @@ public override Uri GetPackageAbsoluteUri(PackageURL purl)
                             Signature = pair[1]
                         });
                     }
-                    
+
                     // size
                     if (OssUtilities.GetJSONPropertyIfExists(distElement, "unpackedSize") is JsonElement sizeElement &&
                         sizeElement.GetInt64() is long size)
@@ -370,7 +371,7 @@ public override Uri GetPackageAbsoluteUri(PackageURL purl)
                     {
                         metadata.Homepage = homepage;
                     }
-                    
+
                     // commit id
                     if (OssUtilities.GetJSONPropertyStringIfExists(versionElement, "gitHead") is string gitHead &&
                         !string.IsNullOrWhiteSpace(gitHead))
@@ -427,18 +428,21 @@ public override Uri GetPackageAbsoluteUri(PackageURL purl)
                     }
 
                     // repository
-                    Dictionary<PackageURL, double> repoMappings = await SearchRepoUrlsInPackageMetadata(purl, content);
-                    foreach (KeyValuePair<PackageURL, double> repoMapping in repoMappings)
+                    if (includeRepositoryMetadata)
                     {
-                        Repository repository = new()
+                        Dictionary<PackageURL, double> repoMappings = await SearchRepoUrlsInPackageMetadata(purl, contentJSON);
+                        foreach (KeyValuePair<PackageURL, double> repoMapping in repoMappings)
                         {
-                            Rank = repoMapping.Value,
-                            Type = repoMapping.Key.Type
-                        };
-                        await repository.ExtractRepositoryMetadata(repoMapping.Key);
+                            Repository repository = new()
+                            {
+                                Rank = repoMapping.Value,
+                                Type = repoMapping.Key.Type
+                            };
+                            await repository.ExtractRepositoryMetadata(repoMapping.Key);
 
-                        metadata.Repository ??= new List<Repository>();
-                        metadata.Repository.Add(repository);
+                            metadata.Repository ??= new List<Repository>();
+                            metadata.Repository.Add(repository);
+                        }
                     }
 
                     // keywords
@@ -483,6 +487,19 @@ is JsonElement.ArrayEnumerator enumeratorElement &&
             return ParseUploadTime(jsonDoc, purl.Version);
         }
 
+        private DateTime? ParseCreatedTime(JsonDocument jsonDoc)
+        {
+            if (jsonDoc.RootElement.TryGetProperty("time", out JsonElement time))
+            {
+                string? createdTime = OssUtilities.GetJSONPropertyStringIfExists(time, "created");
+                if (createdTime != null)
+                {
+                    return DateTime.Parse(createdTime).ToUniversalTime();
+                }
+            }
+            return null;
+        }
+
         private DateTime? ParseUploadTime(JsonDocument jsonDoc, string versionKey)
         {
             if (jsonDoc.RootElement.TryGetProperty("time", out JsonElement time))

src/Shared/PackageManagers/NuGetProjectManager.cs

@@ -183,7 +183,7 @@ public async Task<bool> GetHasReservedNamespaceAsync(PackageURL purl, bool useCa
         }
 
         /// <inheritdoc />
-        public override async Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true)
+        public override async Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true, bool includeRepositoryMetadata = true)
         {
             string? latestVersion = await Actions.GetLatestVersionAsync(purl, includePrerelease: includePrerelease, useCache: useCache);
 
@@ -214,7 +214,7 @@ public async Task<bool> GetHasReservedNamespaceAsync(PackageURL purl, bool useCa
             metadata.LatestPackageVersion = latestVersion;
 
             // Get the metadata for either the specified package version, or the latest package version
-            await UpdateVersionMetadata(metadata, packageVersionMetadata);
+            await UpdateVersionMetadata(metadata, packageVersionMetadata, includeRepositoryMetadata);
 
             return metadata;
         }
@@ -250,7 +250,7 @@ public override async Task<bool> PackageVersionExistsAsync(PackageURL purl, bool
         /// </summary>
         /// <param name="metadata">The <see cref="PackageMetadata"/> object to update with the values for this version.</param>
         /// <param name="packageVersionMetadata">The <see cref="NuGetPackageVersionMetadata"/> representing this version.</param>
-        private async Task UpdateVersionMetadata(PackageMetadata metadata, NuGetPackageVersionMetadata packageVersionMetadata)
+        private async Task UpdateVersionMetadata(PackageMetadata metadata, NuGetPackageVersionMetadata packageVersionMetadata, bool includeRepositoryMetadata)
         {
             if (metadata.PackageVersion is null)
             {
@@ -273,7 +273,10 @@ private async Task UpdateVersionMetadata(PackageMetadata metadata, NuGetPackageV
             UpdateMetadataAuthorsAndMaintainers(metadata, packageVersionMetadata);
 
             // Repository
-            await UpdateMetadataRepository(metadata);
+            if(includeRepositoryMetadata)
+            {
+                await UpdateMetadataRepository(metadata);
+            }
 
             // Dependencies
             IList<PackageDependencyGroup> dependencyGroups = packageVersionMetadata.DependencySets.ToList();

src/Shared/PackageManagers/PyPIProjectManager.cs

@@ -297,7 +297,7 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
 
         /// <inheritdoc />
         /// <remarks>Currently doesn't respect the <paramref name="includePrerelease"/> flag.</remarks>
-        public override async Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true)
+        public override async Task<PackageMetadata?> GetPackageMetadataAsync(PackageURL purl, bool includePrerelease = false, bool useCache = true, bool includeRepositoryMetadata = true)
         {
             PackageMetadata metadata = new();
             string? content = await GetMetadataAsync(purl, useCache);
@@ -345,18 +345,21 @@ public override async Task<IEnumerable<string>> EnumerateVersionsAsync(PackageUR
             metadata.Maintainers.Add(maintainer);
 
             // repository
-            Dictionary<PackageURL, double>? repoMappings = await SearchRepoUrlsInPackageMetadata(purl, content);
-            foreach (KeyValuePair<PackageURL, double> repoMapping in repoMappings)
+            if (includeRepositoryMetadata)
             {
-                Repository repository = new()
+                Dictionary<PackageURL, double>? repoMappings = await SearchRepoUrlsInPackageMetadata(purl, content);
+                foreach (KeyValuePair<PackageURL, double> repoMapping in repoMappings)
                 {
-                    Rank = repoMapping.Value,
-                    Type = repoMapping.Key.Type
-                };
-                await repository.ExtractRepositoryMetadata(repoMapping.Key);
+                    Repository repository = new()
+                    {
+                        Rank = repoMapping.Value,
+                        Type = repoMapping.Key.Type
+                    };
+                    await repository.ExtractRepositoryMetadata(repoMapping.Key);
 
-                metadata.Repository ??= new List<Repository>();
-                metadata.Repository.Add(repository);
+                    metadata.Repository ??= new List<Repository>();
+                    metadata.Repository.Add(repository);
+                }
             }
 
             // license

src/oss-tests/ProjectManagerTests/NPMProjectManagerTests.cs

@@ -337,7 +337,38 @@ public async Task GetPublishedAtSucceeds(string purlString, string? expectedTime
                 Assert.AreEqual(DateTime.Parse(expectedTime), time);
             }
         }
-
+
+        [DataTestMethod]
+        [DataRow("pkg:npm/[email protected]", "2012-04-23T16:37:11.912")]
+        [DataRow("pkg:npm/%40angular/[email protected]", "2016-04-28T04:23:30.108")]
+        [DataRow("pkg:npm/[email protected]", "2018-08-06T12:04:34.792")]
+        [DataRow("pkg:npm/[email protected]", "2018-12-19T23:29:18.197")]
+        [DataRow("pkg:npm/[email protected]", "2022-03-04T05:57:01.108")]
+        public async Task GetCreatedAtSucceeds(string purlString, string? expectedTime = null)
+        {
+            PackageURL purl = new(purlString);
+            var metadata = await _projectManager.Object.GetPackageMetadataAsync(purl, useCache: false);
+            Assert.AreEqual(DateTime.Parse(expectedTime), metadata.CreatedTime);
+        }
+
+        [DataTestMethod]
+        [DataRow(true)]
+        [DataRow(false)]
+        public async Task FetchesRepositoryMetadataSuccessfully(bool includeRepositoryMetadata)
+        {
+            PackageURL purl = new("pkg:npm/lodash.js");
+            var metadata = await _projectManager.Object.GetPackageMetadataAsync(purl, includeRepositoryMetadata: includeRepositoryMetadata);
+
+            if(includeRepositoryMetadata)
+            {
+                Assert.IsNotNull(metadata.Repository);
+            }
+            else
+            {
+                Assert.IsNull(metadata.Repository);
+            }
+        }
+
         [DataTestMethod]
         [DataRow("pkg:npm/[email protected]", "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz")]
         [DataRow("pkg:npm/%40angular/[email protected]", "https://registry.npmjs.org/%40angular/core/-/core-13.2.5.tgz")]

src/oss-tests/ProjectManagerTests/NuGetProjectManagerTests.cs

@@ -299,7 +299,15 @@ public async Task GetPackagePrefixReservedSucceeds(string purlString, bool expec
 
             Assert.AreEqual(expectedReserved, isReserved);
         }
-
+
+        public async Task SkipsRepositoryMetadataFetchSuccessfully()
+        {
+            PackageURL purl = new("pkg:nuget/[email protected]");
+            var metadata = await _projectManager.GetPackageMetadataAsync(purl, includeRepositoryMetadata: false);
+
+            Assert.IsNull(metadata.Repository);
+        }
+
         [DataTestMethod]
         [DataRow("pkg:nuget/[email protected]", 
             "https://api.nuget.org/v3-flatcontainer/newtonsoft.json/13.0.1/newtonsoft.json.13.0.1.nupkg",

src/oss-tests/ProjectManagerTests/PyPIProjectManagerTests.cs

@@ -191,6 +191,25 @@ public async Task DetailedPackageExistsAsync_WorksAsExpected(string purlString,
             Assert.AreEqual(exists, existence.Exists);
         }
 
+
+        [DataTestMethod]
+        [DataRow(true)]
+        [DataRow(false)]
+        public async Task FetchesRepositoryMetadataSuccessfully(bool includeRepositoryMetadata)
+        {
+            PackageURL purl = new("pkg:pypi/[email protected]");
+            var metadata = await _projectManager.GetPackageMetadataAsync(purl, includeRepositoryMetadata: includeRepositoryMetadata);
+
+            if (includeRepositoryMetadata)
+            {
+                Assert.IsNotNull(metadata.Repository);
+            }
+            else
+            {
+                Assert.IsNull(metadata.Repository);
+            }
+        }
+
         [DataTestMethod]
         [DataRow("pkg:pypi/[email protected]", true)]
         [DataRow("pkg:pypi/[email protected]", false)]