feat(terraformer): added terraformer integration (#4686)

Signed-off-by: João Reigota <[email protected]>

.github/workflows/go-ci.yml

@@ -18,7 +18,7 @@ jobs:
         uses: golangci/[email protected]
         with:
           version: v1.37
-          args: -c .golangci.yml
+          args: -c .golangci.yml --timeout 10m
           skip-go-installation: true
   go-generate:
     name: go-generate
@@ -68,6 +68,13 @@ jobs:
       - name: Get Modules
         run: |
           go mod vendor
+      - name: Set Windows Page size
+        if: matrix.os == 'windows-latest'
+        uses: al-cheb/[email protected]
+        with:
+          minimum-size: 32GB
+          maximum-size: 32GB
+          disk-root: "C:"
       - name: Test and Generate Report
         if: matrix.os != 'windows-latest'
         run: |

Dockerfile

@@ -35,8 +35,14 @@ HEALTHCHECK CMD wget -q --method=HEAD localhost/system-status.txt
 # kics-scan ignore-line
 FROM alpine:3.14.3
 
-# Install Git
-RUN apk add --no-cache \
+# Install Terraform and Terraform plugins
+RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip \
+    && unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip \
+    && mv terraform /usr/bin/terraform \
+    && wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip \
+    && unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip \
+    && mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 ~/.terraform.d/plugins/linux_amd64 \
+    && apk add --no-cache \
     git=2.32.0-r0
 
 # Copy built binary to the runtime container

Dockerfile.apispec

@@ -33,6 +33,16 @@ FROM alpine:3.14.3
 
 RUN addgroup -S Checkmarx && adduser -S Checkmarx -G Checkmarx
 
+# Install Terraform and Terraform plugins
+RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip
+RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip
+RUN mv terraform /usr/bin/terraform
+
+RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip
+RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip
+RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 ~/.terraform.d/plugins/linux_amd64
+
+
 # Install Git
 RUN apk add --no-cache \
   git=2.32.0-r0

Dockerfile.apispec.debian

@@ -46,13 +46,21 @@ RUN groupadd checkmarx && useradd -g checkmarx -M -s /bin/bash checkmarx
 
 ENV PATH /app/bin:/usr/bin/git:$PATH
 
-# Install Git
-RUN apt-get update -yq
-
-RUN apt-get install git -y \
+RUN apt-get update -yq \
+  && apt-get install git wget unzip zip -y \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
+# Install Terraform and Terraform plugins
+RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip
+RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip
+RUN mv terraform /usr/bin/terraform
+
+RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip
+RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip
+RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 ~/.terraform.d/plugins/linux_amd64
+
+
 COPY --from=build_env /app/bin/kics /app/bin/kics
 COPY --from=build_env /app/assets/libraries/common /app/bin/assets/libraries/common
 COPY --from=build_env /app/assets/libraries/openapi /app/bin/assets/libraries/openapi

Dockerfile.debian

@@ -47,13 +47,21 @@ RUN groupadd checkmarx && useradd -g checkmarx -M -s /bin/bash checkmarx
 
 ENV PATH /app/bin:/usr/bin/git:$PATH
 
-# Install Git
-RUN apt-get update -yq
-
-RUN apt-get install git -y \
+RUN apt-get update -yq \
+  && apt-get install git wget unzip zip -y \
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/*
 
+# Install Terraform and Terraform plugins
+RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip
+RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip
+RUN mv terraform /usr/bin/terraform
+
+RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip
+RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip
+RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 ~/.terraform.d/plugins/linux_amd64
+
+
 COPY --from=build_env /app/bin/kics /app/bin/kics
 COPY --from=build_env /app/assets/queries /app/bin/assets/queries
 COPY --from=build_env /app/assets/libraries/* /app/bin/assets/libraries/

Dockerfile.ubi7

@@ -54,7 +54,7 @@ ARG KGROUP=kics
 ARG UID=1000
 ARG GID=1000
 
-RUN yum install git -y \
+RUN yum install git wget unzip -y \
   && groupadd -g ${UID} ${KGROUP} \
   && adduser \
   --home-dir /app/bin \
@@ -64,12 +64,26 @@ RUN yum install git -y \
   -s /bin/bash \
   ${KUSER}
 
+
 USER ${KUSER}
 
 COPY LICENSE /licenses/LICENSE
 
 WORKDIR /app/bin
 
+USER root
+
+# Install Terraform and Terraform plugins
+RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip
+RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip
+RUN mv terraform /usr/bin/terraform
+
+RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip
+RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip
+RUN mkdir /app/bin/.terraform.d && mkdir /app/bin/.terraform.d/plugins && mkdir /app/bin/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 /app/bin/.terraform.d/plugins/linux_amd64
+
+USER ${KUSER}
+
 # Copy built binary to the runtime container
 COPY --chown=${KUSER}:${KGROUP} --from=build_env /build/bin/kics /app/bin/kics
 COPY --chown=${KUSER}:${KGROUP} --from=build_env /build/assets/ /app/bin/assets/

docs/integrations.md

@@ -16,6 +16,8 @@ Integrate KICS with:
 - [TeamCity](integrations_teamcity.md)
 - [Travis](integrations_travisci.md)
 - [Pre-commit hooks](integrations_pre_commit.md)
+- [Terraform Cloud](integration_tfcloud.md)
+- [Terraformer](integrations_terraformer.md)
 - More soon...
 
 The pipelines examples can be found in our [GitHub Repository](https://github.com/Checkmarx/kics/tree/master/examples)
@@ -26,9 +28,9 @@ KICS is natively embedded in [MegaLinter](https://megalinter.github.io/), a 100%
 
 - 48 languages
 - 22 formats
-- 20 tooling formats 
-- excessive copy-pastes and spelling mistakes 
- 
+- 20 tooling formats
+- excessive copy-pastes and spelling mistakes
+
 It also generates various reports, and can apply formatting and auto-fixes, to ensure all your projects sources are clean, whatever IDE/toolbox are used by their developers.
 
 To install MegaLinter in your repository, just run the following command

docs/integrations_terraformer.md

@@ -0,0 +1,180 @@
+# Running KICS with Terraformer
+
+From version 1.5, KICS integrates with Terraformer to scan resources deployed in the Cloud. The runtime information of the resources is obtained by providing a Terraformer path to KICS, via `-p` flag. The scan happens immediately after this information is obtained. In the end, results are shown as for any other KICS scan.
+
+**Cloud providers supported:**
+- AWS
+
+## Configure AWS Credentials
+
+For KICS to get the runtime information of your resources you need to provide AWS account Credentials as environment variables. Please note the AWS account provided should have read permissions to list service resources.
+
+Setting AWS credentials as environment variables
+
+MacOS and Linux:
+```sh
+export AWS_ACCESS_KEY_ID="<AWS_ACCESS_KEY_ID>"
+export AWS_SECRET_ACCESS_KEY="<AWS_SECRET_ACCESS_KEY>"
+export AWS_SESSION_TOKEN="<AWS_SESSION_TOKEN>"
+```
+
+Windows:
+
+```sh
+SET AWS_ACCESS_KEY_ID=<AWS_ACCESS_KEY_ID>
+SET AWS_SECRET_ACCESS_KEY=<AWS_SECRET_ACCESS_KEY>
+SET AWS_SESSION_TOKEN=<AWS_SESSION_TOKEN>
+```
+
+Powershell:
+
+```sh
+$Env:AWS_ACCESS_KEY_ID="<AWS_ACCESS_KEY_ID>"
+$Env:AWS_SECRET_ACCESS_KEY="<AWS_SECRET_ACCESS_KEY>"
+$Env:AWS_SESSION_TOKEN="<AWS_SESSION_TOKEN>"
+```
+
+
+
+## KICS Terraformer Path Syntax
+
+```sh
+terraformer::{CloudProvider}:{Resources}:{Regions}
+```
+
+**CloudProvider**: The name of the Cloud Provider to import from.
+
+Possible values:
+- `aws`
+
+**Resources:** A slash-separated list of the resources intended to be imported and scanned.
+
+You can find a complete list of possible values [here](https://github.com/GoogleCloudPlatform/terraformer/blob/master/docs/aws.md#supported-services)
+
+To import all resources please use: `*`
+
+**Regions**: A slash-separated list of the regions to import from.
+
+## Running KICS with Terraformer
+
+When Running KICS using a terraformer path, resources are imported using the credentials set as environment variables in terraform format to the current working directory in a new folder named `kics-extract-terraformer` following the above-described structure.
+KICS will then run a scan on these local files.
+
+If the flag `-o, --output-path` is passed the folder `kics-extract-terraformer` will be generated in the reports directory instead.
+
+### Imported Resources tree structure:
+
+```
+ ▾ kics-extract-terraformer/
+    ▾ {region}/
+        ▾ {resource}/
+            provider.tf
+            {resource}.tf
+            terraform.tfstate
+            variables.tf
+```
+
+### Docker
+
+To run KICS Terraformer integration with Docker simply pass the AWS Credentials that were set as environment variables to the `docker run` command and use the terraformer path syntax
+
+Examples:
+
+```sh
+docker run -e AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID -e AWS_SESSION_TOKEN checkmarx/kics:latest scan -p "terraformer::aws:vpc:eu-west-2" -v --no-progress
+```
+```sh
+docker run -e AWS_SECRET_ACCESS_KEY -e AWS_ACCESS_KEY_ID -e AWS_SESSION_TOKEN -v ${PWD}:/path/ checkmarx/kics:latest scan -p "terraformer::aws:vpc:eu-west-2" -v --no-progress -o /path/results
+```
+
+
+
+<img src="./img/docker_terraformer.gif" />
+
+### Executable
+
+
+### **Disclaimer:** In order to run terraformer with KICS executable please follow these prerequisites:
+
+### Install Terraform
+
+Follow the steps described in Hashicorp documentation https://learn.hashicorp.com/tutorials/terraform/install-cli#install-terraform to install terraform.
+
+### Install AWS Provider Plugin
+
+It is required that the AWS Provider plugin for terraform to be present.
+
+To install AWS Provider plugin:
+- Download the plugin from [Terraform Providers](https://releases.hashicorp.com/terraform-provider-aws/3.72.0/) according to your architecture.
+- Unzip the file to:
+
+### Linux:
+```
+$HOME/.terraform.d/plugins/linux_{arch}/
+
+Example:
+~/.terraform.d/plugins/linux_amd64/terraform-provider-aws_v3.71.0_x5
+```
+
+### MacOS
+
+```
+$HOME/.terraform.d/plugins/darwin_{arch}
+
+Example:
+$HOME/.terraform.d/plugins/darwin_amd64/terraform-provider-aws_3.72.0_darwin_amd64
+```
+
+### Windows:
+
+For Windows a little more work is required, since you can't globally install the AWS Provider plugin, you need to have it present in every directory you wish to import the resources to.
+
+Please follow these steps:
+
+- Create a versions.tf file in the folder you wish to run KICS and import the resources to.
+
+- Paste the code found under `USE PROVIDER` from terraform AWS Provider [Documentation](https://registry.terraform.io/providers/hashicorp/aws/latest/docs) in the versions.tf file you just created.
+
+- run the command `terraform init` on the directory containing `versions.tf`. A new folder named `.terraform` should have been created containing the plugin. This folder must be present in every directory you wish to run KICS on using terraformer.
+
+**NOTE:** `.terraform.hcl.lock` can be deleted
+
+Example tf file:
+
+```hcl
+terraform {
+  required_providers {
+    aws = {
+      source = "hashicorp/aws"
+      version = "3.72.0"
+    }
+  }
+}
+
+provider "aws" {
+  # Configuration options
+}
+```
+
+## Examples:
+
+Example path:
+
+```sh
+kics scan -p 'terraformer::aws:vpc/subnet:eu-west-2/eu-west-1'
+```
+
+These examples showcase KICS integration with terraformer for importing and scanning our VPCs in region `eu-west-2`.
+
+### Linux
+
+<img src="./img/linux_terraformer.gif" />
+
+### Windows
+
+<img src="./img/windows_terraformer.gif" />
+
+## **NOTES**
+
+- If environment credentials are incorrect a timeout may occur.
+- If the resource to import doesn't exist in the region specified a {resource}.tf file will not be created.

go.mod

@@ -3,7 +3,9 @@ module github.com/Checkmarx/kics
 go 1.16
 
 require (
+	cloud.google.com/go/monitoring v1.2.0 // indirect
 	github.com/BurntSushi/toml v0.4.1
+	github.com/GoogleCloudPlatform/terraformer v0.8.18
 	github.com/agnivade/levenshtein v1.1.1
 	github.com/alexmullins/zip v0.0.0-20180717182244-4affb64b04d0
 	github.com/antlr/antlr4/runtime/Go/antlr v0.0.0-20211114212643-ec144ca0d701
@@ -38,6 +40,9 @@ require (
 	helm.sh/helm/v3 v3.7.1
 )
 
-replace github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.0.2
-
-replace github.com/docker/cli => github.com/docker/cli v20.10.12+incompatible
+replace (
+	github.com/docker/cli => github.com/docker/cli v20.10.12+incompatible
+	github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.0.2
+	github.com/spf13/afero => github.com/spf13/afero v1.2.2
+	gopkg.in/jarcoal/httpmock.v1 => github.com/jarcoal/httpmock v1.0.5
+)