Improve data collection performance and output

src/SdnDiagnostics.psd1

@@ -59,19 +59,24 @@
         'Debug-SdnLoadBalancerMux',
         'Debug-SdnNetworkController',
         'Debug-SdnServer',
+        'Disable-SdnDiagTraceOutputLogging',
         'Disable-SdnRasGatewayTracing',
+        'Enable-SdnDiagTraceOutputLogging',
         'Enable-SdnRasGatewayTracing',
-        'Enable-SdnVipTrace'
+        'Enable-SdnVipTrace',
         'Get-SdnAuditLog',
         'Get-SdnApiEndpoint'
         'Get-SdnCertificate',
         'Get-SdnConfigState',
         'Get-SdnDiagnosticLogFile',
+        'Get-SdnDiagTraceOutputLogging',
+        'Get-SdnDiagTraceOutputFile',
         'Get-SdnEventLog',
         'Get-SdnFabricInfrastructureResult',
         'Get-SdnGateway',
         'Get-SdnInfrastructureInfo',
         'Get-SdnInternalLoadBalancer',
+        'Get-SdnLogFile',
         'Get-SdnModuleConfiguration',
         'Get-SdnMuxCertificate',
         'Get-SdnMuxDistributedRouterIP',

src/modules/SdnDiag.Common.Config.psd1

@@ -2,6 +2,8 @@
 # Licensed under the MIT License.
 
 @{
+    Name = "Common"
+    WindowsFeature = @()
     DefaultLogDirectory = "C:\Windows\Tracing\SDNDiagnostics"
     DefaultLogFolders = @(
         "CrashDumps",
@@ -12,14 +14,15 @@
         "Traces"
     )
     LogFileTypes = @(
-        "*.log",
-        "*.etl",
+        "*.blg",
         "*.cab",
         "*.dmp"
+        "*.etl",
+        "*.json",
+        "*.log",
         "*.trace"
         "*.zip"
-        "*.blg"
-        )
+    )
     Properties = @{
         EventLogProviders = @(
             "Application"

src/modules/SdnDiag.Gateway.psm1

@@ -40,31 +40,28 @@ function Get-GatewayConfigState {
     $currentErrorActionPreference = $ErrorActionPreference
     $ProgressPreference = 'SilentlyContinue'
     $ErrorActionPreference = 'Ignore'
+    [string]$outDir = Join-Path -Path $OutputDirectory.FullName -ChildPath "ConfigState\Gateway"
 
     try {
         $config = Get-SdnModuleConfiguration -Role 'Gateway'
-        [System.IO.FileInfo]$OutputDirectory = Join-Path -Path $OutputDirectory.FullName -ChildPath "ConfigState"
-        [System.IO.FileInfo]$regDir = Join-Path -Path $OutputDirectory.FullName -ChildPath "Registry"
-
         "Collect configuration state details for role {0}" -f $config.Name | Trace-Output
-
-        if (-NOT (Initialize-DataCollection -Role 'Gateway' -FilePath $OutputDirectory.FullName -MinimumMB 100)) {
+        if (-NOT (Initialize-DataCollection -Role 'Gateway' -FilePath $outDir -MinimumMB 100)) {
             "Unable to initialize environment for data collection" | Trace-Output -Level:Error
             return
         }
 
-        Export-RegistryKeyConfigDetails -Path $config.properties.regKeyPaths -OutputDirectory $regDir.FullName
-        Get-CommonConfigState -OutputDirectory $OutputDirectory.FullName
+        [string]$regDir = Join-Path -Path $outDir -ChildPath "Registry"
+        Export-RegistryKeyConfigDetails -Path $config.properties.regKeyPaths -OutputDirectory $regDir
 
         # dump out the role configuration state properties
         "Getting RRAS VPN configuration details" | Trace-Output -Level:Verbose
-        Get-VpnServerConfiguration | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-VpnServerConfiguration' -FileType txt -Format Table
-        Get-VpnS2SInterface | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-VpnS2SInterface' -FileType txt -Format List
-        Get-RemoteaccessRoutingDomain | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-RemoteaccessRoutingDomain' -FileType txt -Format List
+        Get-VpnServerConfiguration | Export-ObjectToFile -FilePath $outDir -FileType txt -Format List
+        Get-VpnS2SInterface | Export-ObjectToFile -FilePath $outDir FileType txt -Format List
+        Get-RemoteaccessRoutingDomain | Export-ObjectToFile -FilePath $outDir -FileType txt -Format List
 
         foreach ($routingDomain in Get-RemoteAccessRoutingDomain) {
             "Getting properties for routing domain {0}" -f $routingDomain.RoutingDomain | Trace-Output -Level:Verbose
-            $routingDomainPath = New-Item -Path (Join-Path -Path $OutputDirectory.FullName -ChildPath $routingDomain.RoutingDomainID) -ItemType Directory -Force
+            $routingDomainPath = New-Item -Path (Join-Path -Path $outDir -ChildPath $routingDomain.RoutingDomainID) -ItemType Directory -Force
             Get-BgpRouter -RoutingDomain $routingDomain.RoutingDomain | Export-ObjectToFile -FilePath $routingDomainPath.FullName -Name 'Get-BgpRouter' -FileType txt -Format List
             Get-BgpPeer -RoutingDomain $routingDomain.RoutingDomain | Export-ObjectToFile -FilePath $routingDomainPath.FullName -Name 'Get-BgpPeer' -FileType txt -Format List
             Get-BgprouteInformation -RoutingDomain $routingDomain.RoutingDomain | Export-ObjectToFile -FilePath $routingDomainPath.FullName -Name 'Get-BgprouteInformation' -FileType txt -Format List
@@ -78,7 +75,7 @@ function Get-GatewayConfigState {
         # for ipsec fast path, there is a new service w/ new cmdlets to get the tunnels and routing domains
         if ((Get-Service -Name 'GatewayService').Status -ieq 'Running') {
             "GatewayService is enabled. Getting GatewayService configuration details" | Trace-Output -Level:Verbose
-            $gatewayServicePath = New-Item -Path (Join-Path -Path $OutputDirectory.FullName -ChildPath 'GatewayService') -ItemType Directory -Force
+            $gatewayServicePath = New-Item -Path (Join-Path -Path $outDir -ChildPath 'GatewayService') -ItemType Directory -Force
             Get-Service -Name 'GatewayService' | Export-ObjectToFile -FilePath $gatewayServicePath.FullName -Prefix 'GatewayService' -Name 'Get-Service' -FileType txt -Format List
             Get-GatewayConfiguration | Export-ObjectToFile -FilePath $gatewayServicePath.FullName -Name 'Get-GatewayConfiguration' -FileType txt -Format List
             Get-GatewayRoutingDomain | Export-ObjectToFile -FilePath $gatewayServicePath.FullName -Name 'Get-GatewayRoutingDomain' -FileType txt -Format List
@@ -88,10 +85,10 @@ function Get-GatewayConfigState {
             foreach ($routingDomain in  Get-GatewayRoutingDomain) {
                 "Getting properties for routing domain {0}" -f $routingDomain.RoutingDomain | Trace-Output -Level:Verbose
                 $routingDomainPath = New-Item -Path (Join-Path -Path $gatewayServicePath.FullName -ChildPath $routingDomain.RoutingDomain) -ItemType Directory -Force
-                Get-BgpRouter -RoutingDomain $routingDomain.RoutingDomain | Export-ObjectToFile -FilePath $routingDomainPath.FullName -Name 'Get-BgpRouter' -FileType txt -Format List
-                Get-BgpPeer -RoutingDomain $routingDomain.RoutingDomain | Export-ObjectToFile -FilePath $routingDomainPath.FullName -Name 'Get-BgpPeer' -FileType txt -Format List
-                Get-BgpRouteInformation -RoutingDomain $routingDomain.RoutingDomain | Export-ObjectToFile -FilePath $routingDomainPath.FullName -Name 'Get-BgpRouteInformation' -FileType txt -Format List
-                Get-BgpCustomRoute -RoutingDomain $routingDomain.RoutingDomain | Export-ObjectToFile -FilePath $routingDomainPath.FullName -Name 'Get-BgpCustomRoute' -FileType txt -Format List
+                Get-BgpRouter -RoutingDomain $routingDomain.RoutingDomain | Export-ObjectToFile -FilePath $routingDomainPath.FullName -FileType txt -Format List
+                Get-BgpPeer -RoutingDomain $routingDomain.RoutingDomain | Export-ObjectToFile -FilePath $routingDomainPath.FullName -FileType txt -Format List
+                Get-BgpRouteInformation -RoutingDomain $routingDomain.RoutingDomain | Export-ObjectToFile -FilePath $routingDomainPath.FullName -FileType txt -Format List
+                Get-BgpCustomRoute -RoutingDomain $routingDomain.RoutingDomain | Export-ObjectToFile -FilePath $routingDomainPath.FullName -FileType txt -Format List
             }
         }
     }

src/modules/SdnDiag.Health.psm1

@@ -2228,7 +2228,7 @@ function Test-VMNetAdapterDuplicateMacAddress {
     $sdnHealthTest = New-SdnHealthTest
 
     try {
-        $vmNetAdapters = Get-SdnVMNetworkAdapter
+        $vmNetAdapters = Get-SdnVMNetworkAdapter -All
         $duplicateObjects = $vmNetAdapters | Group-Object -Property MacAddress | Where-Object { $_.Count -ge 2 }
         if ($duplicateObjects) {
             $sdnHealthTest.Result = 'FAIL'

src/modules/SdnDiag.LoadBalancerMux.psm1

@@ -90,31 +90,28 @@ function Get-SlbMuxConfigState {
     $currentErrorActionPreference = $ErrorActionPreference
     $ProgressPreference = 'SilentlyContinue'
     $ErrorActionPreference = 'Ignore'
+    [string]$outDir = Join-Path -Path $OutputDirectory.FullName -ChildPath "ConfigState/LoadBalancerMux"
 
     try {
         $config = Get-SdnModuleConfiguration -Role 'LoadBalancerMux'
-        [System.IO.FileInfo]$OutputDirectory = Join-Path -Path $OutputDirectory.FullName -ChildPath "ConfigState"
-        [System.IO.FileInfo]$regDir = Join-Path -Path $OutputDirectory.FullName -ChildPath "Registry"
-
         "Collect configuration state details for role {0}" -f $config.Name | Trace-Output
-
-        if (-NOT (Initialize-DataCollection -Role $config.Name -FilePath $OutputDirectory.FullName -MinimumMB 100)) {
+        if (-NOT (Initialize-DataCollection -Role $config.Name -FilePath $outDir -MinimumMB 20)) {
             "Unable to initialize environment for data collection" | Trace-Output -Level:Error
             return
         }
 
-        Export-RegistryKeyConfigDetails -Path $config.properties.regKeyPaths -OutputDirectory $regDir.FullName
-        Get-CommonConfigState -OutputDirectory $OutputDirectory.FullName
+        [string]$regDir = Join-Path -Path $outDir -ChildPath "Registry"
+        Export-RegistryKeyConfigDetails -Path $config.properties.regKeyPaths -OutputDirectory $regDir
 
         # output slb configuration and states
         "Getting MUX Driver Control configuration settings" | Trace-Output -Level:Verbose
-        Get-SdnMuxState | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-SdnMuxState' -FileType json
-        Get-SdnMuxDistributedRouterIP | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-SdnMuxDistributedRouterIP' -FileType json
-        Get-SdnMuxStatefulVip | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-SdnMuxStatefulVip' -FileType json
-        Get-SdnMuxStatelessVip | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-SdnMuxStatelessVip' -FileType json
-        Get-SdnMuxStats | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-SdnMuxStats' -FileType json
-        Get-SdnMuxVip | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-SdnMuxVip' -FileType json
-        Get-SdnMuxVipConfig | Export-ObjectToFile -FilePath $OutputDirectory.FullName -Name 'Get-SdnMuxVipConfig' -FileType json
+        Get-SdnMuxState | Export-ObjectToFile -FilePath $outDir -FileType txt -Format List
+        Get-SdnMuxDistributedRouterIP | Export-ObjectToFile -FilePath $outDir -FileType txt -Format List
+        Get-SdnMuxStatefulVip | Export-ObjectToFile -FilePath $outDir -FileType txt -Format List
+        Get-SdnMuxStatelessVip | Export-ObjectToFile -FilePath $outDir -FileType txt -Format List
+        Get-SdnMuxStats | Export-ObjectToFile -FilePath $outDir -FileType txt -Format List
+        Get-SdnMuxVip | Export-ObjectToFile -FilePath $outDir -FileType txt -Format List
+        Get-SdnMuxVipConfig | Export-ObjectToFile -FilePath $outDir -FileType txt -Format List
     }
     catch {
         $_ | Trace-Exception

src/modules/SdnDiag.NetworkController.Config.psd1

@@ -6,9 +6,7 @@
     WindowsFeature = @(
         "NetworkController"
     )
-    RequiredModules = @(
-        "NetworkController"
-    )
+    RequiredModules = @()
     Properties = @{
         apiResources = @{
             AccessControlLists = @{

src/modules/SdnDiag.NetworkController.FC.Config.psd1

@@ -6,14 +6,10 @@
     WindowsFeature = @(
         "NetworkController"
     )
-    RequiredModules = @(
-        "NetworkController"
-    )
+    RequiredModules = @()
     Properties = @{
         CommonPaths = @{}
         EventLogProviders = @(
-            'Microsoft-Windows-FailoverClustering*',
-            'Microsoft-Windows-FailoverClustering-Manager*',
             'NetworkControllerFc'
         )
         RegKeyPaths = @(

src/modules/SdnDiag.NetworkController.FC.psm1

@@ -41,20 +41,17 @@ function Get-NetworkControllerFCConfigState {
     $currentErrorActionPreference = $ErrorActionPreference
     $ProgressPreference = 'SilentlyContinue'
     $ErrorActionPreference = 'SilentlyContinue'
+    [string]$outDir = Join-Path -Path $OutputDirectory.FullName -ChildPath "ConfigState/NetworkController"
 
     try {
         $config = Get-SdnModuleConfiguration -Role 'NetworkController_FC'
-        [string]$outDir = Join-Path -Path $OutputDirectory.FullName -ChildPath "ConfigState"
-        [string]$regDir = Join-Path -Path $OutputDirectory.FullName -ChildPath "Registry"
-
+        "Collect configuration state details for role {0}" -f $config.Name | Trace-Output
         if (-NOT (Initialize-DataCollection -Role $config.Name -FilePath $outDir -MinimumMB 10)) {
             "Unable to initialize environment for data collection for {0}" -f $config.Name | Trace-Output -Level:Error
             return
         }
 
-        "Collect configuration state details for role {0}" -f $config.Name | Trace-Output
-
-        # collect registry configuration information
+        [string]$regDir = Join-Path -Path $outDir -ChildPath "Registry"
         Export-RegistryKeyConfigDetails -Path $config.properties.regKeyPaths -OutputDirectory $regDir
     }
     catch {
@@ -181,15 +178,15 @@ function Get-SdnNetworkControllerFCClusterInfo {
             $clusterName = Get-SdnClusterName -NetworkController $NetworkController -Credential $Credential -ErrorAction Stop
         }
 
-        Get-Cluster -Name $clusterName | Export-ObjectToFile -FilePath $outputDir -Name 'Get-Cluster' -FileType json
-        Get-ClusterFaultDomain -CimSession $clusterName | Export-ObjectToFile -FilePath $outputDir -Name 'Get-ClusterFaultDomain' -FileType json
-        Get-ClusterNode -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -Name 'Get-ClusterNode' -FileType json
-        Get-ClusterGroup -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -Name 'Get-ClusterGroup' -FileType json
-        Get-ClusterNetwork -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -Name 'Get-ClusterNetwork' -FileType json
-        Get-ClusterNetworkInterface -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -Name 'Get-ClusterNetworkInterface' -FileType json
-        Get-ClusterResource -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -Name 'Get-ClusterResource' -FileType json
-        Get-ClusterResourceType -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -Name 'Get-ClusterResourceType' -FileType txt -Format Table
-        Get-ClusterSharedVolume -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -Name 'Get-ClusterSharedVolume' -FileType json
+        Get-Cluster -Name $clusterName | Export-ObjectToFile -FilePath $outputDir -FileType txt -Format List
+        Get-ClusterFaultDomain -CimSession $clusterName | Export-ObjectToFile -FilePath $outputDir -FileType txt -Format List
+        Get-ClusterNode -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -FileType txt -Format List
+        Get-ClusterGroup -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -FileType txt -Format List
+        Get-ClusterNetwork -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -FileType txt -Format List
+        Get-ClusterNetworkInterface -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -FileType txt -Format List
+        Get-ClusterResource -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -FileType txt -Format List
+        Get-ClusterResourceType -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -FileType txt -Format List
+        Get-ClusterSharedVolume -Cluster $clusterName | Export-ObjectToFile -FilePath $outputDir -FileType txt -Format List
     }
     catch {
         $_ | Trace-Exception
@@ -294,3 +291,28 @@ function Confirm-IsFailoverClusterNC {
 
     return $false
 }
+
+function Get-SdnClusterLog {
+    [CmdletBinding()]
+    param(
+        [Parameter(Position = 0)]
+        [System.IO.FileInfo]$OutputDirectory = (Get-WorkingDirectory)
+    )
+
+    # The 3>$null 4>$null sends warning and error to null
+    # typically Get-ClusterLog does not like remote powershell operations and generates warnings/errors
+    $clusterLogFiles = Get-ClusterLog -Destination $OutputDirectory.FullName 2>$null 3>$null
+
+    # if we have cluster log files, we will zip them up to preserve disk space
+    if ($clusterLogFiles) {
+        $clusterLogFiles | ForEach-Object {
+            $zipFilePath = Join-Path -Path $OutputDirectory.FullName -ChildPath ($_.Name + ".zip")
+            Compress-Archive -Path $_.FullName -DestinationPath $zipFilePath -Force -ErrorAction Stop
+
+            # if the file was successfully zipped, we can remove the original file
+            if (Get-Item -Path $zipFilePath -ErrorAction Ignore) {
+                Remove-Item -Path $_.FullName -Force -ErrorAction Ignore
+            }
+        }
+    }
+}

src/modules/SdnDiag.NetworkController.SF.Config.psd1

@@ -6,9 +6,7 @@
     WindowsFeature = @(
         "NetworkController"
     )
-    RequiredModules = @(
-        "NetworkController"
-    )
+    RequiredModules = @()
     Properties = @{
         CommonPaths = @{
             serviceFabricLogDirectory = @(