feat(engine): semantic exit code based on results (#2400) (#1721)

.github/workflows/go-ci-integration.yml

@@ -50,7 +50,7 @@ jobs:
       run: |
         docker run -v ${PWD}/assets/queries:/path \
           -e SENTRY_DSN=${{secrets.SENTRY_DSN}} \
-          kics:${{ github.sha }} --silent --log-level DEBUG --log-file --log-path "/path/info.log" -p "/path" -o "/path/results.json"
+          kics:${{ github.sha }} --silent --ignore-on-exit "results" --log-level DEBUG --log-file --log-path "/path/info.log" -p "/path" -o "/path/results.json"
     - name: Archive test logs
       uses: actions/upload-artifact@v2
       if: always()

cmd/console/main.go

@@ -4,10 +4,14 @@ import (
 	"os"
 
 	"github.com/Checkmarx/kics/internal/console"
+	"github.com/Checkmarx/kics/internal/console/helpers"
+	"github.com/Checkmarx/kics/internal/constants"
 )
 
 func main() { // nolint:funlen,gocyclo
 	if err := console.Execute(); err != nil {
-		os.Exit(1)
+		if helpers.ShowError("errors") {
+			os.Exit(constants.EngineErrorCode)
+		}
 	}
 }

docs/results.md

@@ -299,3 +299,22 @@ SARIF reports are sorted by severity (from high to info), following [SARIF v2.1.
 HTML reports are sorted by severity (from high to info), the results will have query information, a list of files which vulnerability was found and a code snippet where the problem was detected as you can see in following example:
 
 <img src="https://raw.githubusercontent.com/Checkmarx/kics/master/docs/img/html_report.png" width="850">
+
+# Exit Status Code
+
+## Results Status Code
+
+| Code          | Description                               |
+| --------------| ------------------------------------------|
+| `0`           | No Results were Found                     |
+| `50`          | Found any `HIGH` Results                  |
+| `40`          | Found any `MEDIUM` Results                |
+| `30`          | Found any `LOW` Results                   |
+| `20`          | Found any `INFO` Results                  |
+
+## Error Status Code
+
+| Code      | Description                                   |
+| --------------| ------------------------------------------|
+| `126`         | Engine Error                              |
+| `130`         | Signal-Interrupt                          |

docs/usage/commands.md

@@ -50,6 +50,12 @@ Flags:
   -x, --exclude-results strings      exclude results by providing the similarity ID of a result
                                      can be provided multiple times or as a comma separated string
                                      example: 'fec62a97d569662093dbb9739360942f...,31263s5696620s93dbb973d9360942fc2a...'
+      --fail-on                      which kind of results should return an exit code different from 0
+                                     accetps: high, medium, low and info
+                                     example: "high,low"
+      --ignoreOnExitFlag             defines which kind of non-zero exits code should be ignored
+                                     accepts: all, results, errors, none
+                                     example: if 'results' is set, only engine errors will make KICS exit code different from 0
   -h, --help                         help for scan
       --minimal-ui                   simplified version of CLI output
       --no-progress                  hides the progress bar

e2e/cli_test.go

@@ -70,7 +70,7 @@ var tests = []struct {
 			},
 			expectedOut: []string{"E2E_CLI_003"},
 		},
-		wantStatus: 1,
+		wantStatus: 126,
 	},
 	// E2E-CLI-004 - KICS scan command had a mandatory flag -p the CLI should exhibit
 	// an error message and return exit code 1
@@ -88,7 +88,7 @@ var tests = []struct {
 				"E2E_CLI_004",
 			},
 		},
-		wantStatus: 1,
+		wantStatus: 126,
 	},
 	// E2E-CLI-005 - KICS scan with -- payload-path flag should create a file with the
 	// passed name containing the payload of the files scanned
@@ -106,7 +106,7 @@ var tests = []struct {
 				"E2E_CLI_005_PAYLOAD",
 			},
 		},
-		wantStatus:    0,
+		wantStatus:    50,
 		removePayload: []string{"payload.json"},
 	},
 }

e2e/fixtures/E2E_CLI_002

@@ -17,7 +17,13 @@ Flags:
   -x, --exclude-results strings      exclude results by providing the similarity ID of a result
                                      can be provided multiple times or as a comma separated string
                                      example: 'fec62a97d569662093dbb9739360942f...,31263s5696620s93dbb973d9360942fc2a...'
+      --fail-on strings              which kind of results should return an exit code different from 0
+                                     accetps: high, medium, low and info
+                                     example: "high,low" (default [high,medium,low,info])
   -h, --help                         help for scan
+      --ignore-on-exit string        defines which kind of non-zero exits code should be ignored
+                                     accepts: all, results, errors, none
+                                     example: if 'results' is set, only engine errors will make KICS exit code different from 0 (default "none")
       --minimal-ui                   simplified version of CLI output
       --no-progress                  hides the progress bar
   -o, --output-path string           directory path to store reports

e2e/fixtures/E2E_CLI_003

@@ -16,7 +16,13 @@ Flags:
   -x, --exclude-results strings      exclude results by providing the similarity ID of a result
                                      can be provided multiple times or as a comma separated string
                                      example: 'fec62a97d569662093dbb9739360942f...,31263s5696620s93dbb973d9360942fc2a...'
+      --fail-on strings              which kind of results should return an exit code different from 0
+                                     accetps: high, medium, low and info
+                                     example: "high,low" (default [high,medium,low,info])
   -h, --help                         help for scan
+      --ignore-on-exit string        defines which kind of non-zero exits code should be ignored
+                                     accepts: all, results, errors, none
+                                     example: if 'results' is set, only engine errors will make KICS exit code different from 0 (default "none")
       --minimal-ui                   simplified version of CLI output
       --no-progress                  hides the progress bar
   -o, --output-path string           directory path to store reports

e2e/fixtures/E2E_CLI_004

@@ -16,7 +16,13 @@ Flags:
   -x, --exclude-results strings      exclude results by providing the similarity ID of a result
                                      can be provided multiple times or as a comma separated string
                                      example: 'fec62a97d569662093dbb9739360942f...,31263s5696620s93dbb973d9360942fc2a...'
+      --fail-on strings              which kind of results should return an exit code different from 0
+                                     accetps: high, medium, low and info
+                                     example: "high,low" (default [high,medium,low,info])
   -h, --help                         help for scan
+      --ignore-on-exit string        defines which kind of non-zero exits code should be ignored
+                                     accepts: all, results, errors, none
+                                     example: if 'results' is set, only engine errors will make KICS exit code different from 0 (default "none")
       --minimal-ui                   simplified version of CLI output
       --no-progress                  hides the progress bar
   -o, --output-path string           directory path to store reports

internal/console/helpers/exit_handler.go

@@ -0,0 +1,71 @@
+package helpers
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/Checkmarx/kics/pkg/model"
+)
+
+var shouldIgnore string
+var shouldFail map[string]struct{}
+
+// ResultsExitCode calculate exit code base on severity of results, returns 0 if no results was reported
+func ResultsExitCode(summary *model.Summary) int {
+	// severityArr is needed to make sure 'for' cycle is made in an ordered fashion
+	severityArr := []model.Severity{"HIGH", "MEDIUM", "LOW", "INFO"}
+	codeMap := map[model.Severity]int{"HIGH": 50, "MEDIUM": 40, "LOW": 30, "INFO": 20}
+	exitMap := summary.SeveritySummary.SeverityCounters
+	for _, severity := range severityArr {
+		if _, reportSeverity := shouldFail[strings.ToLower(string(severity))]; !reportSeverity {
+			continue
+		}
+		if exitMap[severity] > 0 {
+			return codeMap[severity]
+		}
+	}
+	return 0
+}
+
+// InitShouldIgnoreArg initializes what kind of errors should be used on exit codes
+func InitShouldIgnoreArg(arg string) error {
+	validArgs := []string{"none", "all", "results", "errors"}
+	for _, validArg := range validArgs {
+		if strings.EqualFold(validArg, arg) {
+			shouldIgnore = strings.ToLower(arg)
+			return nil
+		}
+	}
+	return fmt.Errorf("unknown argument for --ignore-on-exit: %s\nvalid arguments:\n  %s", arg, strings.Join(validArgs, "\n  "))
+}
+
+// InitShouldFailArg initializes which kind of vulnerability severity should changes exit code
+func InitShouldFailArg(args []string) error {
+	possibleArgs := map[string]struct{}{
+		"high":   {},
+		"medium": {},
+		"low":    {},
+		"info":   {},
+	}
+	if len(args) == 0 {
+		shouldFail = possibleArgs
+		return nil
+	}
+
+	argsConverted := make(map[string]struct{})
+	for _, arg := range args {
+		if _, ok := possibleArgs[strings.ToLower(arg)]; !ok {
+			validArgs := []string{"high", "medium", "low", "info"}
+			return fmt.Errorf("unknown argument for --fail-on: %s\nvalid arguments:\n  %s", arg, strings.Join(validArgs, "\n  "))
+		}
+		argsConverted[strings.ToLower(arg)] = struct{}{}
+	}
+
+	shouldFail = argsConverted
+	return nil
+}
+
+// ShowError returns true if should show error, otherwise returns false
+func ShowError(kind string) bool {
+	return strings.EqualFold(shouldIgnore, "none") || (!strings.EqualFold(shouldIgnore, "all") && !strings.EqualFold(shouldIgnore, kind))
+}