Counting generic queries with aggregation value on metadata (#2127)

assets/queries/ansible/azure/sensitive_port_is_exposed_to_entire_network/metadata.json

@@ -5,5 +5,6 @@
 	"category": "Network Ports Security",
 	"descriptionText": "A sensitive port, such as port 23 or port 110, is open for the whole network in either TCP or UDP protocol",
 	"descriptionUrl": "https://docs.ansible.com/ansible/latest/collections/azure/azcollection/azure_rm_securitygroup_module.html#parameter-rules",
-	"platform": "Ansible"
+	"platform": "Ansible",
+  "aggregation": 35
 }

assets/queries/ansible/azure/sensitive_port_is_exposed_to_entire_network/query.rego

@@ -1,5 +1,7 @@
 package Cx
 
+import data.generic.ansible as lib
+
 getFieldName(field) = name {
 	upper(field) == "NETWORK PORTS SECURITY"
 	name := "azure_rm_securitygroup"
@@ -110,7 +112,7 @@ CxPolicy[result] {
 
 	#############	document and resource
 	document := input.document[i]
-	tasks := getTasks(document)
+	tasks := lib.getTasks(document)
 	task := tasks[t]
 
 	resource := task[field].rules[r]
@@ -139,11 +141,3 @@ CxPolicy[result] {
 		"keyActualValue": sprintf("%s (%s:%d) is allowed in %s.%s.rules", [portName, protocol, portNumber, ruleName, field]),
 	}
 }
-
-getTasks(document) = result {
-	result := [body | playbook := document.playbooks[0]; body := playbook.tasks]
-	count(result) != 0
-} else = result {
-	result := [body | playbook := document.playbooks[_]; body := playbook]
-	count(result) != 0
-}

assets/queries/cloudFormation/ec2_sensitive_port_is_publicly_exposed/metadata.json

@@ -5,5 +5,6 @@
 	"category": "Network Ports Security",
 	"descriptionText": "The EC2 instance has a sensitive port connection exposed to the entire network",
 	"descriptionUrl": "https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-security-group.html",
-	"platform": "CloudFormation"
+	"platform": "CloudFormation",
+  "aggregation": 61
 }

assets/queries/cloudFormation/elb_sensitive_port_is_exposed_to_entire_network/metadata.json

@@ -5,5 +5,6 @@
 	"category": "Network Ports Security",
 	"descriptionText": "The load balancer of the application with a sensitive port connection is exposed to the entire internet.",
 	"descriptionUrl": "https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-ec2-elb.html",
-	"platform": "CloudFormation"
+	"platform": "CloudFormation",
+  "aggregation": 183
 }

assets/queries/terraform/aws/sensitive_port_is_exposed_to_entire_network/metadata.json

@@ -5,5 +5,6 @@
 	"category": "Network Ports Security",
 	"descriptionText": "A sensitive port, such as port 23 or port 110, is open for the whole network in either TCP or UDP protocol",
 	"descriptionUrl": "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group",
-	"platform": "Terraform"
+	"platform": "Terraform",
+  "aggregation": 48
 }

assets/queries/terraform/aws/sensitive_port_is_exposed_to_small_public_network/metadata.json

@@ -5,5 +5,6 @@
 	"category": "Network Ports Security",
 	"descriptionText": "A sensitive port, such as port 23 or port 110, is open for a small public network in either TCP or UDP protocol",
 	"descriptionUrl": "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group",
-	"platform": "Terraform"
+	"platform": "Terraform",
+  "aggregation": 48
 }

assets/queries/terraform/azure/sensitive_port_is_exposed_to_entire_network/metadata.json

@@ -5,5 +5,6 @@
 	"category": "Network Ports Security",
 	"descriptionText": "A sensitive port, such as port 23 or port 110, is open for the whole network in either TCP or UDP protocol",
 	"descriptionUrl": "https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_rule",
-	"platform": "Terraform"
+	"platform": "Terraform",
+  "aggregation": 48
 }

assets/queries/terraform/azure/sensitive_port_is_exposed_to_small_public_network/metadata.json

@@ -5,5 +5,6 @@
   "category": "Network Ports Security",
   "descriptionText": "A sensitive port, such as port 23 or port 110, is open for small public network in either TCP or UDP protocol",
   "descriptionUrl": "https://registry.terraform.io/providers/hashicorp/azurerm/latest/docs/resources/network_security_rule",
-  "platform": "Terraform"
+  "platform": "Terraform",
+  "aggregation": 48
 }

internal/tracker/ci.go

@@ -11,13 +11,13 @@ type CITracker struct {
 }
 
 // TrackQueryLoad adds a loaded query
-func (c *CITracker) TrackQueryLoad() {
-	c.LoadedQueries++
+func (c *CITracker) TrackQueryLoad(queryAggregation int) {
+	c.LoadedQueries += queryAggregation
 }
 
 // TrackQueryExecution adds a query executed
-func (c *CITracker) TrackQueryExecution() {
-	c.ExecutedQueries++
+func (c *CITracker) TrackQueryExecution(queryAggregation int) {
+	c.ExecutedQueries += queryAggregation
 }
 
 // TrackFileFound adds a found file to be scanned

internal/tracker/ci_test.go

@@ -44,12 +44,12 @@ func TestCITracker(t *testing.T) {
 			FailedSimilarityID: tt.fields.FailedSimilarityID,
 		}
 		t.Run(fmt.Sprintf(tt.name+"_LoadedQueries"), func(t *testing.T) {
-			c.TrackQueryLoad()
+			c.TrackQueryLoad(1)
 			require.Equal(t, 1, c.LoadedQueries)
 		})
 
 		t.Run(fmt.Sprintf(tt.name+"_TrackQueryExecution"), func(t *testing.T) {
-			c.TrackQueryExecution()
+			c.TrackQueryExecution(1)
 			require.Equal(t, 1, c.ExecutedQueries)
 		})
 

pkg/engine/inspector.go

@@ -52,8 +52,8 @@ type QueriesSource interface {
 // TrackQueryExecution increments the number of queries executed
 // FailedDetectLine decrements the number of queries executed
 type Tracker interface {
-	TrackQueryLoad()
-	TrackQueryExecution()
+	TrackQueryLoad(queryAggregation int)
+	TrackQueryExecution(queryAggregation int)
 	FailedDetectLine()
 	FailedComputeSimilarityID()
 }
@@ -145,7 +145,7 @@ func NewInspector(
 				continue
 			}
 
-			tracker.TrackQueryLoad()
+			tracker.TrackQueryLoad(metadata.Aggregation)
 
 			opaQueries = append(opaQueries, &preparedQuery{
 				opaQuery: opaQuery,
@@ -155,8 +155,10 @@ func NewInspector(
 	}
 	failedQueries := make(map[string]error)
 
+	queriesNumber := sumAllAggregatedQueries(opaQueries)
+
 	log.Info().
-		Msgf("Inspector initialized, number of queries=%d\n", len(opaQueries))
+		Msgf("Inspector initialized, number of queries=%d\n", queriesNumber)
 
 	return &Inspector{
 		queries:        opaQueries,
@@ -167,6 +169,14 @@ func NewInspector(
 	}, nil
 }
 
+func sumAllAggregatedQueries(opaQueries []*preparedQuery) int {
+	sum := 0
+	for _, query := range opaQueries {
+		sum += query.metadata.Aggregation
+	}
+	return sum
+}
+
 func startProgressBar(hideProgress bool, total int, wg *sync.WaitGroup, progressChannel chan float64) {
 	wg.Add(1)
 	progressBar := consoleHelpers.NewProgressBar("Executing queries: ", 10, float64(total), progressChannel)
@@ -222,7 +232,7 @@ func (c *Inspector) Inspect(
 
 		vulnerabilities = append(vulnerabilities, vuls...)
 
-		c.tracker.TrackQueryExecution()
+		c.tracker.TrackQueryExecution(query.metadata.Aggregation)
 	}
 	close(currentQuery)
 	wg.Wait()

pkg/engine/inspector_test.go

@@ -335,6 +335,7 @@ func TestNewInspector(t *testing.T) { // nolint
 				"descriptionUrl":  "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket#acl",
 				"platform":        "CloudFormation",
 			},
+			Aggregation: 1,
 		},
 	})
 	type args struct {

pkg/engine/query/source.go

@@ -143,11 +143,17 @@ func ReadQuery(queryDir string) (model.QueryMetadata, error) {
 	metadata := ReadMetadata(queryDir)
 	platform := getPlatform(queryDir)
 
+	aggregation := 1
+	if agg, ok := metadata["aggregation"]; ok {
+		aggregation = int(agg.(float64))
+	}
+
 	return model.QueryMetadata{
-		Query:    path.Base(filepath.ToSlash(queryDir)),
-		Content:  string(queryContent),
-		Metadata: metadata,
-		Platform: platform,
+		Query:       path.Base(filepath.ToSlash(queryDir)),
+		Content:     string(queryContent),
+		Metadata:    metadata,
+		Platform:    platform,
+		Aggregation: aggregation,
 	}, nil
 }
 

pkg/engine/query/source_test.go

@@ -196,7 +196,8 @@ func TestFilesystemSource_GetQueries(t *testing.T) {
 						"severity":        "HIGH",
 						"platform":        "CloudFormation",
 					},
-					Platform: "unknown",
+					Platform:    "unknown",
+					Aggregation: 1,
 				},
 			},
 			wantErr: false,
@@ -218,7 +219,12 @@ func TestFilesystemSource_GetQueries(t *testing.T) {
 				t.Errorf("FilesystemSource.GetQueries() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			require.Equal(t, tt.want, got)
+			wantStr, err := test.StringifyStruct(tt.want)
+			require.Nil(t, err)
+			gotStr, err := test.StringifyStruct(got)
+			require.Nil(t, err)
+
+			require.Equal(t, tt.want, got, "want = %s\ngot = %s", wantStr, gotStr)
 		})
 	}
 }
@@ -256,13 +262,18 @@ func Test_ReadMetadata(t *testing.T) {
 				"queryName":       "<QUERY_NAME>",
 				"severity":        "HIGH",
 				"platform":        "<PLATFORM>",
+				"aggregation":     float64(1),
 			},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			if got := ReadMetadata(tt.args.queryDir); !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("readMetadata() = %v, want %v", got, tt.want)
+				gotStr, err := test.StringifyStruct(got)
+				require.Nil(t, err)
+				wantStr, err := test.StringifyStruct(tt.want)
+				require.Nil(t, err)
+				t.Errorf("readMetadata()\ngot = %v\nwant = %v", gotStr, wantStr)
 			}
 		})
 	}

pkg/model/model.go

@@ -70,6 +70,9 @@ type QueryMetadata struct {
 	Content  string
 	Metadata map[string]interface{}
 	Platform string
+	// special field for generic queries
+	// represents how many queries are aggregated into a single rego file
+	Aggregation int
 }
 
 // Vulnerability is a representation of a detected vulnerability in scanned files

test/fixtures/type-test01/template01/metadata.json

@@ -5,5 +5,6 @@
   "category": null,
   "descriptionText": "<TEXT>",
   "descriptionUrl": "#",
-  "platform": "<PLATFORM>"
+  "platform": "<PLATFORM>",
+  "aggregation": 1
 }