Add "EnableNonBacktrackingRegex" option to oss-characteristic tool (#453

) * Add "EnableNonBacktrackingRegex" option to oss-characteristic tool Also requires bumping .net version for related projects to net 8 to pick up the non-backtracking behavior that requires 7+. * Update projects to net 6 and 8 * Update pipelines for net 8 * Update dependencies * Options for Backtrack and Single Thread Adds options for using the default backtracking engine and to use single threaded app inspector to oss-characteristics and oss-detect-backdoor.
microsoft · Jan 9, 2024 · 7c0c3a7 · 7c0c3a7
1 parent 16f88ed
commit 7c0c3a7
Show file tree

Hide file tree

Showing 23 changed files with 168 additions and 78 deletions.
diff --git a/Pipelines/templates/dotnet-build-publish-all-platforms-job.yml b/Pipelines/templates/dotnet-build-publish-all-platforms-job.yml
@@ -10,7 +10,11 @@ parameters:
 # Version of Dotnet SDK to use
 - name: dotnetVersion
   type: string
-  default: '6.0.x'
+  default: '8.0.x'
+# Version of Dotnet to publish
+- name: dotnetPublishVersion
+  type: string
+  default: 'net8.0'
 # Include preview versions of Dotnet SDK
 - name: includePreviewVersions
   type: boolean
@@ -67,28 +71,28 @@ jobs:
     displayName: Publish Linux x64
     inputs:
       command: 'publish'
-      arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/linux/${{ parameters.projectName }}_linux_$(ReleaseVersion) --sc -r linux-x64'
+      arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/linux/${{ parameters.projectName }}_linux_$(ReleaseVersion) --sc -r linux-x64 -f ${{ parameters.dotnetPublishVersion }}'
       publishWebProjects: false
       zipAfterPublish: false
   - task: DotNetCoreCLI@2
     displayName: Publish MacOS x64
     inputs:
       command: 'publish'
-      arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/macos/${{ parameters.projectName }}_macos_$(ReleaseVersion) --sc -r osx-x64'
+      arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/macos/${{ parameters.projectName }}_macos_$(ReleaseVersion) --sc -r osx-x64 -f ${{ parameters.dotnetPublishVersion }}'
       publishWebProjects: false
       zipAfterPublish: false
   - task: DotNetCoreCLI@2
     displayName: Publish Win x64
     inputs:
       command: 'publish'
-      arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/win/${{ parameters.projectName }}_win_$(ReleaseVersion) --sc -r win-x64'
+      arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/win/${{ parameters.projectName }}_win_$(ReleaseVersion) --sc -r win-x64 -f ${{ parameters.dotnetPublishVersion }}'
       publishWebProjects: false
       zipAfterPublish: false
   - task: DotNetCoreCLI@2
     displayName: Build .NET Core App
     inputs:
       command: 'publish'
-      arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/netcoreapp/${{ parameters.projectName }}_netcoreapp_$(ReleaseVersion)'
+      arguments: '${{ parameters.solutionPath }} -c ${{ parameters.buildConfiguration }} -o bin/netcoreapp/${{ parameters.projectName }}_netcoreapp_$(ReleaseVersion) -f ${{ parameters.dotnetPublishVersion }}'
       publishWebProjects: false
       zipAfterPublish: false
   - task: ArchiveFiles@2

diff --git a/Pipelines/templates/dotnet-test-job.yml b/Pipelines/templates/dotnet-test-job.yml
@@ -6,7 +6,7 @@ parameters:
 # Version of Dotnet SDK to use
 - name: dotnetVersion
   type: string
-  default: '6.0.x'
+  default: '8.0.x'
 # Should Dotnet SDK install preview versions?
 - name: includePreviewVersions
   type: boolean

diff --git a/Pipelines/templates/nuget-build-job.yml b/Pipelines/templates/nuget-build-job.yml
@@ -6,7 +6,7 @@ parameters:
 # Version of Dotnet SDK to use
 - name: dotnetVersion
   type: string
-  default: '6.0.x'
+  default: '8.0.x'
 # Should Dotnet SDK install preview versions?
 - name: includePreviewVersions
   type: boolean

diff --git a/src/Shared.CLI/Shared.CLI.csproj b/src/Shared.CLI/Shared.CLI.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
     <RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
     <Description>OSS Gadget - Shared CLI Functionality</Description>
     <RepositoryType>GitHub</RepositoryType>
@@ -39,19 +39,19 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="AngleSharp" Version="1.0.0-alpha-844" />
+    <PackageReference Include="AngleSharp" Version="1.0.7" />
     <PackageReference Include="CommandLineParser" Version="2.9.1" />
     <PackageReference Include="Crayon" Version="2.0.69" />
-    <PackageReference Include="F23.StringSimilarity" Version="5.0.0" />
-    <PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="4.0.1" />
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
-    <PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.1.18" />
-    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="6.0.1" />
-    <PackageReference Include="NLog" Version="5.0.4" />
-    <PackageReference Include="NLog.Schema" Version="5.0.4" />
-    <PackageReference Include="NuGet.Versioning" Version="6.6.1" />
-    <PackageReference Include="Octokit" Version="4.0.1" />
-    <PackageReference Include="Sarif.Sdk" Version="3.1.0" />
+    <PackageReference Include="F23.StringSimilarity" Version="6.0.0" />
+    <PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="4.1.0" />
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+    <PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.2.23" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
+    <PackageReference Include="NLog" Version="5.2.8" />
+    <PackageReference Include="NLog.Schema" Version="5.2.8" />
+    <PackageReference Include="NuGet.Versioning" Version="6.8.0" />
+    <PackageReference Include="Octokit" Version="9.1.0" />
+    <PackageReference Include="Sarif.Sdk" Version="4.4.0" />
     <PackageReference Include="SemanticVersioning" Version="2.0.2" />
     <PackageReference Include="System.Console" Version="4.3.1" />
   </ItemGroup>
@@ -64,4 +64,8 @@
   <ItemGroup>
     <ProjectReference Include="..\Shared\Shared.Lib.csproj" />
   </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
+  </ItemGroup>
 </Project>
diff --git a/src/Shared/Shared.Lib.csproj b/src/Shared/Shared.Lib.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
 	<PropertyGroup>
-		<TargetFramework>net6.0</TargetFramework>
+		<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
 		<RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
 		<Description>OSS Gadget - Shared Library Functionality</Description>
 		<RepositoryType>GitHub</RepositoryType>
@@ -29,25 +29,25 @@
 	</PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="AngleSharp" Version="1.0.0-alpha-844" />
+        <PackageReference Include="AngleSharp" Version="1.0.7" />
         <PackageReference Include="CommandLineParser" Version="2.9.1" />
         <PackageReference Include="Crayon" Version="2.0.69" />
-        <PackageReference Include="F23.StringSimilarity" Version="5.0.0" />
-        <PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
-        <PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.1.18" />
-        <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="6.0.1" />
-        <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="6.0.1" />
-        <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0" />
-        <PackageReference Include="Microsoft.Extensions.Http" Version="6.0.0" />
-        <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="6.0.10" />
-        <PackageReference Include="NLog" Version="5.0.4" />
-        <PackageReference Include="NLog.Schema" Version="5.0.4" />
-        <PackageReference Include="NuGet.Packaging" Version="6.6.1" />
-        <PackageReference Include="NuGet.Protocol" Version="6.6.1" />
-        <PackageReference Include="Octokit" Version="4.0.1" />
+        <PackageReference Include="F23.StringSimilarity" Version="6.0.0" />
+        <PackageReference Include="HtmlAgilityPack" Version="1.11.57" />
+        <PackageReference Include="Microsoft.CST.RecursiveExtractor" Version="1.2.23" />
+        <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
+        <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
+        <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" />
+        <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
+        <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.0" />
+        <PackageReference Include="NLog" Version="5.2.8" />
+        <PackageReference Include="NLog.Schema" Version="5.2.8" />
+        <PackageReference Include="NuGet.Packaging" Version="6.8.0" />
+        <PackageReference Include="NuGet.Protocol" Version="6.8.0" />
+        <PackageReference Include="Octokit" Version="9.1.0" />
         <PackageReference Include="packageurl-dotnet" Version="1.3.0" />
         <PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
-        <PackageReference Include="Sarif.Sdk" Version="3.1.0" />
+        <PackageReference Include="Sarif.Sdk" Version="4.4.0" />
         <PackageReference Include="SemanticVersioning" Version="2.0.2" />
         <PackageReference Include="System.Console" Version="4.3.1" />
         <PackageReference Include="System.Linq.Async" Version="6.0.1" />
@@ -64,4 +64,8 @@
         <None Include="..\..\icon-128.png" Pack="true" PackagePath="" />
     </ItemGroup>
 
+    <ItemGroup>
+      <PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
+    </ItemGroup>
+
 </Project>
diff --git a/src/oss-characteristics/CharacteristicTool.cs b/src/oss-characteristics/CharacteristicTool.cs
@@ -66,6 +66,12 @@ public static IEnumerable<Example> Examples
                 HelpText = "exclude files or paths which match provided glob patterns.")]
             public string FilePathExclusions { get; set; } = "";
 
+            [Option('b', "backtracking", Required = false, HelpText = "Use backtracking regex engine by default.")]
+            public bool EnableBacktracking { get; set; } = false;
+
+            [Option('s', "single-threaded", Required = false, HelpText = "Use single-threaded analysis")]
+            public bool SingleThread { get; set; } = false;
+
             public bool AllowTagsInBuildFiles { get; set; } = true;
 
             public bool AllowDupTags { get; set; } = false;
@@ -107,8 +113,9 @@ public CharacteristicTool() : this(new ProjectManagerFactory())
                 ConfidenceFilters = new [] { Confidence.High | Confidence.Medium | Confidence.Low },
                 ScanUnknownTypes = true,
                 AllowAllTagsInBuildFiles = options.AllowTagsInBuildFiles,
-                SingleThread = false,
-                FilePathExclusions = options.FilePathExclusions?.Split(',') ?? Array.Empty<string>()
+                SingleThread = options.SingleThread,
+                FilePathExclusions = options.FilePathExclusions?.Split(',') ?? Array.Empty<string>(),
+                EnableNonBacktrackingRegex = !options.EnableBacktracking
             };
 
             try

diff --git a/src/oss-characteristics/oss-characteristic.csproj b/src/oss-characteristics/oss-characteristic.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
     <RootNamespace>Microsoft.CST.OpenSource</RootNamespace> 
     <Description>OSS Gadget - Characteristic Identifier</Description>
     <RepositoryType>GitHub</RepositoryType>
@@ -29,7 +29,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.CST.ApplicationInspector.Commands" Version="1.6.24" />
+    <PackageReference Include="Microsoft.CST.ApplicationInspector.Commands" Version="1.9.17" />
   </ItemGroup>
 
   <ItemGroup>
@@ -41,4 +41,8 @@
     <None Include="..\..\LICENSE.txt" Pack="true" PackagePath="" />
     <None Include="..\..\icon-128.png" Pack="true" PackagePath="" />
   </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
+  </ItemGroup>
 </Project>
diff --git a/src/oss-defog/oss-defog.csproj b/src/oss-defog/oss-defog.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
     <RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
     <Description>OSS Gadget - Obfuscated String Detector</Description>
     <RepositoryType>GitHub</RepositoryType>
@@ -46,4 +46,8 @@
     <None Include="..\..\LICENSE.txt" Pack="true" PackagePath="" />
     <None Include="..\..\icon-128.png" Pack="true" PackagePath="" />
   </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
+  </ItemGroup>
 </Project>
diff --git a/src/oss-detect-backdoor/DetectBackdoorTool.cs b/src/oss-detect-backdoor/DetectBackdoorTool.cs
@@ -49,6 +49,12 @@ public static IEnumerable<Example> Examples
             [Option('c', "use-cache", Required = false, Default = false,
                 HelpText = "do not download the package if it is already present in the destination directory.")]
             public bool UseCache { get; set; }
+
+            [Option('b', "backtracking", Required = false, HelpText = "Use backtracking engine by default.")]
+            public bool EnableBacktracking { get; set; } = false;
+
+            [Option('s', "single-threaded", Required = false, HelpText = "Use single-threaded analysis")]
+            public bool SingleThread { get; set; } = false;
         }
 
         public DetectBackdoorTool(ProjectManagerFactory projectManagerFactory) : base(projectManagerFactory)
@@ -177,7 +183,9 @@ void WriteMatch(MatchRecord match, int index, int matchCount)
                     AllowTagsInBuildFiles = true,
                     FilePathExclusions = ".md,LICENSE,.txt",
                     AllowDupTags = true,
-                    SarifLevel = CodeAnalysis.Sarif.FailureLevel.Warning
+                    SarifLevel = CodeAnalysis.Sarif.FailureLevel.Warning,
+                    EnableBacktracking = options.EnableBacktracking,
+                    SingleThread = options.SingleThread
                 };
 
                 return await characteristicTool.RunAsync(cOptions);

diff --git a/src/oss-detect-backdoor/oss-detect-backdoor.csproj b/src/oss-detect-backdoor/oss-detect-backdoor.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
     <RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
     <Description>OSS Gadget - Backdoor Identifier</Description>
     <RepositoryType>GitHub</RepositoryType>
@@ -83,4 +83,8 @@
     <None Include="..\..\LICENSE.txt" Pack="true" PackagePath="" />
     <None Include="..\..\icon-128.png" Pack="true" PackagePath="" />
   </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
+  </ItemGroup>
 </Project>
diff --git a/src/oss-detect-cryptography/oss-detect-cryptography.csproj b/src/oss-detect-cryptography/oss-detect-cryptography.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
     <RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
     <Description>OSS Gadget - Cryptography Detector</Description>
     <RepositoryType>GitHub</RepositoryType>
@@ -70,11 +70,11 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="ELFSharp" Version="2.15.0" />
-    <PackageReference Include="ICSharpCode.Decompiler" Version="7.2.1.6856" />
-    <PackageReference Include="Microsoft.CST.ApplicationInspector.Commands" Version="1.6.24" />
-    <PackageReference Include="Microsoft.CST.DevSkim" Version="0.6.32" />
-    <PackageReference Include="PeNet" Version="2.9.9" />
+    <PackageReference Include="ELFSharp" Version="2.17.3" />
+    <PackageReference Include="ICSharpCode.Decompiler" Version="8.2.0.7535" />
+    <PackageReference Include="Microsoft.CST.ApplicationInspector.Commands" Version="1.9.17" />
+    <PackageReference Include="Microsoft.CST.DevSkim" Version="1.0.27" />
+    <PackageReference Include="PeNet" Version="4.0.4" />
     <PackageReference Include="SharpDisasm" Version="1.1.11" />
     <PackageReference Include="WebAssembly" Version="1.2.1" />
   </ItemGroup>
@@ -88,4 +88,8 @@
     <None Include="..\..\LICENSE.txt" Pack="true" PackagePath="" />
     <None Include="..\..\icon-128.png" Pack="true" PackagePath="" />
   </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
+  </ItemGroup>
 </Project>
diff --git a/src/oss-diff/oss-diff.csproj b/src/oss-diff/oss-diff.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
     <RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
     <Description>OSS Gadget - Package Differ</Description>
     <RepositoryType>GitHub</RepositoryType>
@@ -28,8 +28,8 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="DiffPlex" Version="1.7.1" />
-    <PackageReference Include="Pastel" Version="3.0.1" />
+    <PackageReference Include="DiffPlex" Version="1.7.2" />
+    <PackageReference Include="Pastel" Version="4.2.0" />
   </ItemGroup>
 
   <ItemGroup>
@@ -40,4 +40,8 @@
     <None Include="..\..\LICENSE.txt" Pack="true" PackagePath="" />
     <None Include="..\..\icon-128.png" Pack="true" PackagePath="" />
   </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
+  </ItemGroup>
 </Project>
diff --git a/src/oss-download/oss-download.csproj b/src/oss-download/oss-download.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
     <RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
     <Description>OSS Gadget - Package Downloader</Description>
     <RepositoryType>GitHub</RepositoryType>
@@ -43,4 +43,8 @@
     <None Include="..\..\LICENSE.txt" Pack="true" PackagePath="" />
     <None Include="..\..\icon-128.png" Pack="true" PackagePath="" />
   </ItemGroup>
+
+  <ItemGroup>
+    <PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
+  </ItemGroup>
 </Project>
diff --git a/src/oss-find-domain-squats/oss-find-domain-squats.csproj b/src/oss-find-domain-squats/oss-find-domain-squats.csproj
@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFrameworks>net6.0;net8.0</TargetFrameworks>
     <RootNamespace>Microsoft.CST.OpenSource</RootNamespace>
     <Company>Microsoft Corporation</Company>
     <Description>OSS Gadget - Domain Typosquatting Detector</Description>
@@ -17,7 +17,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Pastel" Version="3.0.1" />
+    <PackageReference Include="Pastel" Version="4.2.0" />
     <PackageReference Include="Whois" Version="3.0.1" />
   </ItemGroup>
 
@@ -26,4 +26,8 @@
     <ProjectReference Include="..\Shared.CLI\Shared.CLI.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <PackageReference Update="Nerdbank.GitVersioning" Version="3.6.133" />
+  </ItemGroup>
+
 </Project>