feat(resolver): added openapi file resolver for json and yaml parsers

Signed-off-by: João Reigota <[email protected]>

assets/queries/ansible/aws/api_gateway_without_configured_authorizer/query.rego

@@ -12,7 +12,7 @@ CxPolicy[result] {
 	ansLib.checkState(apiGateway)
 
 	content_info := get_content(apiGateway)
-    
+
 	securityScheme := content_info.content.components.securitySchemes[x]
 	not common_lib.valid_key(securityScheme, "x-amazon-apigateway-authorizer")
 
@@ -68,7 +68,7 @@ without_authorizer(apiGateway) {
 }
 
 get_content(apiGateway) = content_info {
-	content := apiGateway.swagger_file.content
+	content := apiGateway.swagger_file
 	content_info := {"content": content, "attribute": "swagger_file"}
 } else = content_info {
 	content := apiGateway.swagger_dict

cmd/console/main.go

@@ -8,7 +8,7 @@ import (
 	"github.com/Checkmarx/kics/internal/constants"
 )
 
-func main() { // nolint:funlen,gocyclo
+func main() {
 	if err := console.Execute(); err != nil {
 		if helpers.ShowError("errors") {
 			os.Exit(constants.EngineErrorCode)

e2e/fixtures/samples/unresolved_openapi/openapi.yaml

@@ -0,0 +1,27 @@
+openapi: "3.0.0"
+info:
+  version: 1.0.0
+  title: Swagger Petstore
+  description: Multi-file boilerplate for OpenAPI Specification.
+  license:
+    name: MIT
+  contact:
+    name: API Support
+    url: http://www.example.com/support
+    email: [email protected]
+servers:
+  - url: http://petstore.swagger.io/v1
+tags:
+  - name: pets
+paths:
+  /pets:
+    $ref: "./resources/pets.yaml"
+  /pets/{petId}:
+    $ref: "./resources/pet.yaml"
+components:
+  parameters:
+    $ref: "./parameters/_index.yaml"
+  schemas:
+    $ref: "./schemas/_index.yaml"
+  responses:
+    $ref: "./responses/_index.yaml"

e2e/fixtures/samples/unresolved_openapi/parameters/_index.yaml

@@ -0,0 +1,7 @@
+# path
+petId:
+  $ref: './path/petId.yaml'
+
+# limit
+limit:
+  $ref: './query/limit.yaml'

e2e/fixtures/samples/unresolved_openapi/parameters/path/petId.yaml

@@ -0,0 +1,6 @@
+name: petId
+in: path
+required: true
+description: The id of the pet to retrieve
+schema:
+  type: string

e2e/fixtures/samples/unresolved_openapi/parameters/query/limit.yaml

@@ -0,0 +1,7 @@
+name: limit
+in: query
+description: How many items to return at one time (max 100)
+required: false
+schema:
+  type: integer
+  format: int32

e2e/fixtures/samples/unresolved_openapi/resources/pet.yaml

@@ -0,0 +1,17 @@
+get:
+  summary: Detail
+  operationId: showPetById
+  description: Info for a specific pet
+  tags:
+    - pets
+  parameters:
+    - $ref: "../parameters/path/petId.yaml"
+  responses:
+    '200':
+      description: Expected response to a valid request
+      content:
+        application/json:
+          schema:
+            $ref: "../schemas/Pet.yaml"
+    default:
+      $ref: "../responses/UnexpectedError.yaml"

e2e/fixtures/samples/unresolved_openapi/resources/pets.yaml

@@ -0,0 +1,34 @@
+get:
+  summary: List
+  operationId: listPets
+  description: List all pets
+  tags:
+  - pets
+  parameters:
+    - $ref: "../parameters/query/limit.yaml"
+  responses:
+    '200':
+      description: A paged array of pets
+      headers:
+        x-next:
+          description: A link to the next page of responses
+          schema:
+            type: string
+      content:
+        application/json:
+          schema:
+            $ref: "../schemas/Pet.yaml"
+    default:
+      $ref: "../responses/UnexpectedError.yaml"
+post:
+  summary: Create
+  operationId: createPets
+  description: Create a pet
+  tags:
+  - pets
+  responses:
+    '201':
+        $ref : "../responses/NullResponse.yaml"
+    default:
+        $ref : "../responses/UnexpectedError.yaml"
+

e2e/fixtures/samples/unresolved_openapi/responses/NullResponse.yaml

@@ -0,0 +1 @@
+description: Null response

e2e/fixtures/samples/unresolved_openapi/responses/UnexpectedError.yaml

@@ -0,0 +1,5 @@
+description: unexpected error
+content:
+  application/json:
+    schema:
+      $ref : "../schemas/Error.yaml"

e2e/fixtures/samples/unresolved_openapi/responses/_index.yaml

@@ -0,0 +1,4 @@
+UnexpectedError:
+  $ref: "./UnexpectedError.yaml"
+NullResponse:
+  $ref: "./NullResponse.yaml"

e2e/fixtures/samples/unresolved_openapi/schemas/Error.yaml

@@ -0,0 +1,10 @@
+type: object
+required:
+  - code
+  - message
+properties:
+  code:
+    type: integer
+    format: int32
+  message:
+    type: string

e2e/fixtures/samples/unresolved_openapi/schemas/Pet.yaml

@@ -0,0 +1,12 @@
+type: object
+required:
+  - id
+  - name
+properties:
+  id:
+    type: integer
+    format: int64
+  name:
+    type: string
+  tag:
+    type: string

e2e/fixtures/samples/unresolved_openapi/schemas/_index.yaml

@@ -0,0 +1,4 @@
+Pet:
+  $ref: "./Pet.yaml"
+Error:
+  $ref: "./Error.yaml"

e2e/testcases/e2e-cli-055_scan_resolve_openapi_files.go

@@ -0,0 +1,24 @@
+package testcases
+
+import "regexp"
+
+// E2E-CLI-055 - Kics scan command with openapi files that are not resolved
+// should resolve openapi files and return results in different files
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should resolve openapi files and return results in different files [E2E-CLI-055]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-p", "/path/e2e/fixtures/samples/unresolved_openapi"},
+			},
+		},
+		WantStatus: []int{50},
+		Validation: func(outputText string) bool {
+			matchQueryPath1, _ := regexp.MatchString(`UnexpectedError.yaml`, outputText)
+			matchQueryPath2, _ := regexp.MatchString(`petId.yaml`, outputText)
+			return matchQueryPath1 && matchQueryPath2
+		},
+	}
+
+	Tests = append(Tests, testSample)
+}

e2e/utils/helper.go

@@ -29,7 +29,7 @@ func RunCommand(kicsArgs []string, useDocker, useMock bool, kicsDockerImage stri
 		source, args = runKicsDev(kicsArgs)
 	}
 
-	cmd := exec.Command(source, args...) //nolint
+	cmd := exec.Command(source, args...)
 	cmd.Env = append(os.Environ(), descriptionServer)
 	stdOutput, err := cmd.CombinedOutput()
 	if err != nil {

go.mod

@@ -386,10 +386,10 @@ require (
 	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
-	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
+	golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
 	gonum.org/v1/gonum v0.7.0 // indirect
 	google.golang.org/api v0.74.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -2102,8 +2102,9 @@ golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e h1:w36l2Uw3dRan1K3TyXriXvY+6T56GNmlKGcqiQUJDfM=
 golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -2213,8 +2214,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U=
-golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618=
+golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
 gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw=
 gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM=

pkg/builder/writer/rego.go

@@ -218,7 +218,7 @@ func format(rules []build.Rule) []RegoRule {
 	return res
 }
 
-func createBlock(rule build.Rule) Block { // nolint:gocyclo
+func createBlock(rule build.Rule) Block {
 	result := Block{}
 	result = resultName(rule, result)
 

pkg/detector/default_detect.go

@@ -1,7 +1,6 @@
 package detector
 
 import (
-	"github.com/Checkmarx/kics/pkg/resolver/file"
 	"strconv"
 	"strings"
 
@@ -18,13 +17,16 @@ type defaultDetectLine struct {
 
 // DetectLine searches vulnerability line if kindDetectLine is not in detectors
 func (d defaultDetectLine) DetectLine(file *model.FileMetadata, searchKey string,
-	_ *zerolog.Logger, outputLines int) model.VulnerabilityLines {
-	lines := d.SplitLines(file.OriginalData)
-	resFiles := d.prepareResolvedFiles(file.ResolvedFiles)
-	foundAtLeastOne := false
-	currentLine := 0
-	adjLines := lines
-	resolvedFile := file.FilePath
+	outputLines int, logwithfields *zerolog.Logger) model.VulnerabilityLines {
+	detector := &DefaultDetectLineResponse{
+		CurrentLine:     0,
+		IsBreak:         false,
+		FoundAtLeastOne: false,
+		Lines:           d.SplitLines(file.OriginalData),
+		ResolvedFile:    file.FilePath,
+		ResolvedFiles:   d.prepareResolvedFiles(file.ResolvedFiles),
+	}
+
 	var extractedString [][]string
 	extractedString = GetBracketValues(searchKey, extractedString, "")
 	sanitizedSubstring := searchKey
@@ -35,30 +37,27 @@ func (d defaultDetectLine) DetectLine(file *model.FileMetadata, searchKey string
 	for _, key := range strings.Split(sanitizedSubstring, ".") {
 		substr1, substr2 := GenerateSubstrings(key, extractedString)
 
-		response := DetectCurrentLine(adjLines, substr1, substr2, currentLine, foundAtLeastOne, resFiles, resolvedFile)
+		detector = detector.DetectCurrentLine(substr1, substr2)
 
-		currentLine = response.CurrentLine
-		foundAtLeastOne = response.FoundAtLeastOne
-		adjLines = response.Lines
-		resolvedFile = response.ResolvedFile
-
-		if response.IsBreak {
+		if detector.IsBreak {
 			break
 		}
 	}
 
-	if foundAtLeastOne {
+	if detector.FoundAtLeastOne {
 		return model.VulnerabilityLines{
-			Line:         currentLine + 1,
-			VulnLines:    GetAdjacentVulnLines(currentLine, outputLines, adjLines),
-			ResolvedFile: resolvedFile,
+			Line:         detector.CurrentLine + 1,
+			VulnLines:    GetAdjacentVulnLines(detector.CurrentLine, outputLines, detector.Lines),
+			ResolvedFile: detector.ResolvedFile,
 		}
 	}
 
+	logwithfields.Warn().Msgf("Failed to detect line, query response %s", searchKey)
+
 	return model.VulnerabilityLines{
 		Line:         undetectedVulnerabilityLine,
 		VulnLines:    []model.CodeLine{},
-		ResolvedFile: resolvedFile,
+		ResolvedFile: detector.ResolvedFile,
 	}
 }
 
@@ -67,7 +66,7 @@ func (d defaultDetectLine) SplitLines(content string) []string {
 	return strings.Split(text, "\n")
 }
 
-func (d defaultDetectLine) prepareResolvedFiles(resFiles map[string]file.ResolvedFile) map[string]model.ResolvedFileSplit {
+func (d defaultDetectLine) prepareResolvedFiles(resFiles map[string]model.ResolvedFile) map[string]model.ResolvedFileSplit {
 	resolvedFiles := make(map[string]model.ResolvedFileSplit)
 	for f, res := range resFiles {
 		resolvedFiles[f] = model.ResolvedFileSplit{

pkg/detector/default_detect_test.go

@@ -143,7 +143,7 @@ func Test_detectLine(t *testing.T) { //nolint
 	for _, tt := range tests {
 		detector := NewDetectLine(tt.feilds.outputLines)
 		t.Run(tt.name, func(t *testing.T) {
-			got := detector.defaultDetector.DetectLine(tt.args.file, tt.args.searchKey, &zerolog.Logger{}, 3)
+			got := detector.defaultDetector.DetectLine(tt.args.file, tt.args.searchKey, 3, &zerolog.Logger{})
 			gotStrVulnerabilities, err := test.StringifyStruct(got)
 			require.Nil(t, err)
 			wantStrVulnerabilities, err := test.StringifyStruct(tt.want)
@@ -154,3 +154,42 @@ func Test_detectLine(t *testing.T) { //nolint
 		})
 	}
 }
+
+func Test_defaultDetectLine_prepareResolvedFiles(t *testing.T) {
+	type args struct {
+		resFiles map[string]model.ResolvedFile
+	}
+	tests := []struct {
+		name string
+		args args
+		want map[string]model.ResolvedFileSplit
+	}{
+		{
+			name: "prepare_resolved_files",
+			args: args{
+				resFiles: map[string]model.ResolvedFile{
+					"file1": {
+						Content: []byte(
+							`content1
+content2`),
+						Path: "testing/file1",
+					},
+				},
+			},
+			want: map[string]model.ResolvedFileSplit{
+				"file1": {
+					Path:  "testing/file1",
+					Lines: []string{"content1", "content2"},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			d := defaultDetectLine{}
+			if got := d.prepareResolvedFiles(tt.args.resFiles); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("prepareResolvedFiles() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}