setup initial e2e tests closes #2848

Signed-off-by: Rogério Peixoto <[email protected]>

.github/workflows/go-ci.yml

@@ -83,7 +83,8 @@ jobs:
         go mod vendor
     - name: Test and Generate Report
       run: |
-        go test -mod=vendor -v ./... -count=1 -coverprofile cover.out 2>&1 | go-junit-report -set-exit-code -go-version ${{ matrix.go-version }} -package-name "github.com/Checkmarx/kics/test" > test-report-${{ matrix.os }}.xml
+        set +o pipeline
+        go test -mod=vendor -v $(go list ./... | grep -v e2e/) -count=1 -coverprofile cover.out 2>&1 | go-junit-report -set-exit-code -go-version ${{ matrix.go-version }} -package-name "github.com/Checkmarx/kics/test" > test-report-${{ matrix.os }}.xml
     - name: Archive unit tests report
       uses: actions/upload-artifact@v2
       with:

.github/workflows/go-e2e.yaml

@@ -0,0 +1,35 @@
+
+name: go-e2e
+
+on:
+  pull_request:
+    branches: [master]
+
+jobs:
+  unit-tests:
+    name: e2e-tests
+    strategy:
+      matrix:
+        go-version: [1.16.x]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Cancel Previous Runs
+      uses: styfle/[email protected]
+      with:
+        access_token: ${{ github.token }}
+    - name: Set up Go 1.x
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Check out code
+      uses: actions/checkout@v2
+      with:
+        persist-credentials: false
+    - name: Build binary
+      run: make build
+    - name: Run E2E Tests
+      env:
+        E2E_KICS_BINARY: ./bin/kics
+      run: |
+        go test "github.com/Checkmarx/kics/e2e" -v

.golangci.yml

@@ -111,6 +111,7 @@ run:
     - docs
     - vendor
     - tools
+    - e2e
 
 # golangci.com configuration
 # https://github.com/golangci/golangci/wiki/Configuration

Makefile

@@ -49,7 +49,7 @@ build-all: lint generate
 
 .PHONY: build
 build: ## go build
-build: lint generate
+build: generate
 	$(call print-target)
 	@go build -o ${TARGET_BIN} \
 		-ldflags "-X github.com/Checkmarx/kics/internal/constants.Version=${VERSION} -X github.com/Checkmarx/kics/internal/constants.SCMCommit=${COMMIT}" \

e2e/cli_test.go

@@ -0,0 +1,24 @@
+package e2e
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+// E2E_CLI_001 - KICS command should display a help text in the CLI when provided with the --help flag and it should describe the available commands plus the global flags
+func Test_E2E_CLI_001(t *testing.T) {
+	kicsPath := getKICSBinaryPath("../bin/kics")
+	actualOutput, err := runCommandAndReturnOutput([]string{kicsPath, "--help"})
+	require.NoError(t, err, "Capture output should not yield an error")
+	actualLines := strings.Split(actualOutput, "\n")
+
+	expectedOutput, err := readFixture("E2E_CLI_001")
+	require.NoError(t, err, "Reading a fixture should not yield an error")
+	expectedLines := strings.Split(expectedOutput, "\n")
+	for idx := range expectedLines {
+		require.Equal(t, expectedLines[idx], actualLines[idx], fmt.Sprintf("Expected output line\n%s is not equal to actual output line\n%s\n line: %d", expectedLines[idx], actualLines[idx], idx))
+	}
+}

e2e/fixtures/E2E_CLI_001

@@ -0,0 +1,23 @@
+Keeping Infrastructure as Code Secure
+
+Usage:
+  kics [command]
+
+Available Commands:
+  generate-id    Generates uuid for query
+  help           Help about any command
+  list-platforms List supported platforms
+  scan           Executes a scan analysis
+  version        Displays the current version
+
+Flags:
+      --ci                  display only log messages to CLI output (mutually exclusive with silent)
+  -h, --help                help for kics
+  -f, --log-format string   determines log format (pretty,json) (default "pretty")
+      --log-level string    determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO")
+      --log-path string     path to log files, (defaults to ${PWD}/info.log)
+      --no-color            disable CLI color output
+  -s, --silent              silence stdout messages (mutually exclusive with verbose and ci)
+  -v, --verbose             write logs to stdout too (mutually exclusive with silent)
+
+Use "kics [command] --help" for more information about a command.

e2e/utils.go

@@ -0,0 +1,43 @@
+package e2e
+
+import (
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+)
+
+func runCommandAndReturnOutput(args []string) (stdout string, err error) {
+	cmd := exec.Command(args[0], args[1:]...) //nolint
+	stdOutput, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	return string(stdOutput), nil
+}
+
+func readFixture(testName string) (string, error) {
+	return readFile(filepath.Join("fixtures", testName))
+}
+
+func readFile(path string) (string, error) {
+	ostat, err := os.Open(filepath.Clean(path))
+	if err != nil {
+		return "", err
+	}
+	bytes, err := io.ReadAll(ostat)
+	if err != nil {
+		return "", err
+	}
+	return string(bytes), nil
+}
+
+func getKICSBinaryPath(path string) string {
+	var rtnPath string
+	if path == "" {
+		rtnPath = os.Getenv("E2E_KICS_BINARY")
+	} else {
+		rtnPath = path
+	}
+	return rtnPath
+}