Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

revert(terraformer): remove terraformer #6937

Merged
merged 4 commits into from
Mar 7, 2024
Merged

revert(terraformer): remove terraformer #6937

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
563df50
remove terraformer
pereiramarco011 Mar 7, 2024
423b5c6
remove paths filter
pereiramarco011 Mar 7, 2024
9d7dca2
remove Terraform from docker image
pereiramarco011 Mar 7, 2024
c54da34
remove unused file
pereiramarco011 Mar 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 0 additions & 6 deletions .github/workflows/go-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,6 @@ jobs:
uses: actions/checkout@v3
with:
persist-credentials: false
- uses: dorny/paths-filter@v2
id: changes
with:
filters: |
src:
- 'pkg/terraformer/**'
- name: Get cache paths
id: go-cache-paths
shell: bash
Expand Down
2 changes: 0 additions & 2 deletions .grype.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ ignore:
name: anchore/scan-action
- package:
location: "/root/.terraform.d/plugins/linux_amd64/terraform-provider-**"
- package:
location: "/usr/bin/terraformer"
- package:
location: "/usr/bin/terraform"
- package:
Expand Down
18 changes: 0 additions & 18 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,24 +43,6 @@ RUN apk update --no-cache \
gcompat~=1.1.0 \
git~=2.43

# Install Terraform and Terraform plugins
RUN wget https://releases.hashicorp.com/terraform/1.5.6/terraform_1.5.6_linux_amd64.zip \
&& unzip terraform_1.5.6_linux_amd64.zip && rm terraform_1.5.6_linux_amd64.zip \
&& mv terraform /usr/bin/terraform \
&& wget https://releases.hashicorp.com/terraform-provider-azurerm/3.71.0/terraform-provider-azurerm_3.71.0_linux_amd64.zip \
&& wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip \
&& wget https://releases.hashicorp.com/terraform-provider-google/4.32.0/terraform-provider-google_4.32.0_linux_amd64.zip \
&& unzip terraform-provider-azurerm_3.71.0_linux_amd64.zip && rm terraform-provider-azurerm_3.71.0_linux_amd64.zip\
&& unzip terraform-provider-google_4.32.0_linux_amd64.zip && rm terraform-provider-google_4.32.0_linux_amd64.zip \
&& unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip \
&& mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-google_v4.32.0_x5 terraform-provider-azurerm_v3.71.0_x5 ~/.terraform.d/plugins/linux_amd64

# Install Terraformer
RUN wget https://github.com/GoogleCloudPlatform/terraformer/releases/download/0.8.24/terraformer-all-linux-amd64 \
&& chmod +x terraformer-all-linux-amd64 \
&& mv terraformer-all-linux-amd64 /usr/bin/terraformer


# Copy built binary to the runtime container
# Vulnerability fixed in latest version of KICS remove when gh actions version is updated
# kics-scan ignore-line
Expand Down
14 changes: 0 additions & 14 deletions docker/Dockerfile.apispec
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,20 +33,6 @@ FROM alpine:3.15.4

RUN addgroup -S Checkmarx && adduser -S Checkmarx -G Checkmarx

# Install Terraform and Terraform plugins
RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip
RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip
RUN mv terraform /usr/bin/terraform

RUN wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip
RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip
RUN wget https://releases.hashicorp.com/terraform-provider-google/4.10.0/terraform-provider-google_4.10.0_linux_amd64.zip
RUN unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip
RUN unzip terraform-provider-google_4.10.0_linux_amd64.zip && rm terraform-provider-google_4.10.0_linux_amd64.zip
RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip
RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-google_v4.10.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64


# Install Git
RUN apk add --no-cache \
git=2.36.3-r0
Expand Down
14 changes: 0 additions & 14 deletions docker/Dockerfile.apispec.debian
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,20 +51,6 @@ RUN apt-get update -yq \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

# Install Terraform and Terraform plugins
RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip
RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip
RUN mv terraform /usr/bin/terraform

RUN wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip
RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip
RUN wget https://releases.hashicorp.com/terraform-provider-google/4.10.0/terraform-provider-google_4.10.0_linux_amd64.zip
RUN unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip
RUN unzip terraform-provider-google_4.10.0_linux_amd64.zip && rm terraform-provider-google_4.10.0_linux_amd64.zip
RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip
RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-google_v4.10.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64


COPY --from=build_env /app/bin/kics /app/bin/kics
COPY --from=build_env /app/assets/libraries/common /app/bin/assets/libraries/common
COPY --from=build_env /app/assets/libraries/openapi /app/bin/assets/libraries/openapi
Expand Down
14 changes: 0 additions & 14 deletions docker/Dockerfile.debian
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,20 +52,6 @@ RUN apt-get update -yq \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*

# Install Terraform and Terraform plugins
RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip
RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip
RUN mv terraform /usr/bin/terraform

RUN wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip
RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip
RUN wget https://releases.hashicorp.com/terraform-provider-google/4.10.0/terraform-provider-google_4.10.0_linux_amd64.zip
RUN unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip
RUN unzip terraform-provider-google_4.10.0_linux_amd64.zip && rm terraform-provider-google_4.10.0_linux_amd64.zip
RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip
RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-google_v4.10.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64


COPY --from=build_env /app/bin/kics /app/bin/kics
COPY --from=build_env /app/assets/queries /app/bin/assets/queries
COPY --from=build_env /app/assets/cwe_csv /app/bin/assets/cwe_csv
Expand Down
17 changes: 0 additions & 17 deletions docker/Dockerfile.ubi8
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,23 +71,6 @@ COPY LICENSE /licenses/LICENSE

WORKDIR /app/bin

USER root

# Install Terraform and Terraform plugins
RUN wget https://releases.hashicorp.com/terraform/1.1.3/terraform_1.1.3_linux_amd64.zip
RUN unzip terraform_1.1.3_linux_amd64.zip && rm terraform_1.1.3_linux_amd64.zip
RUN mv terraform /usr/bin/terraform

RUN wget https://releases.hashicorp.com/terraform-provider-azurerm/2.95.0/terraform-provider-azurerm_2.95.0_linux_amd64.zip
RUN wget https://releases.hashicorp.com/terraform-provider-aws/3.72.0/terraform-provider-aws_3.72.0_linux_amd64.zip
RUN wget https://releases.hashicorp.com/terraform-provider-google/4.10.0/terraform-provider-google_4.10.0_linux_amd64.zip
RUN unzip terraform-provider-azurerm_2.95.0_linux_amd64.zip && rm terraform-provider-azurerm_2.95.0_linux_amd64.zip
RUN unzip terraform-provider-google_4.10.0_linux_amd64.zip && rm terraform-provider-google_4.10.0_linux_amd64.zip
RUN unzip terraform-provider-aws_3.72.0_linux_amd64.zip && rm terraform-provider-aws_3.72.0_linux_amd64.zip
RUN mkdir ~/.terraform.d && mkdir ~/.terraform.d/plugins && mkdir ~/.terraform.d/plugins/linux_amd64 && mv terraform-provider-aws_v3.72.0_x5 terraform-provider-google_v4.10.0_x5 terraform-provider-azurerm_v2.95.0_x5 ~/.terraform.d/plugins/linux_amd64

USER ${KUSER}

# Copy built binary to the runtime container
COPY --chown=${KUSER}:${KGROUP} --from=build_env /build/bin/kics /app/bin/kics
COPY --chown=${KUSER}:${KGROUP} --from=build_env /build/assets/ /app/bin/assets/
Expand Down
2 changes: 1 addition & 1 deletion docs/integrations_terraformer.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# Running KICS with Terraformer
# Running KICS with Terraformer (Deprecated after 1.7.13)

From version 1.5, KICS integrates with Terraformer to scan resources deployed in the Cloud. The runtime information of the resources is obtained by providing a Terraformer path to KICS, via `-p` flag. The scan happens immediately after this information is obtained. In the end, results are shown as for any other KICS scan.

Expand Down
1 change: 0 additions & 1 deletion docs/release_checklist.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,3 @@
- git checkout integrations
- git tag ${latest_tag}-integrations
- git push origin ${latest_tag}-integrations
NOTE: make sure that `pkg/terraformer` only has `terraformer_alt.go` without build tags
27 changes: 0 additions & 27 deletions pkg/engine/provider/extract.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (

"github.com/Checkmarx/kics/pkg/kuberneter"
"github.com/Checkmarx/kics/pkg/model"
"github.com/Checkmarx/kics/pkg/terraformer"
"github.com/Checkmarx/kics/pkg/utils"
"github.com/rs/zerolog/log"

Expand Down Expand Up @@ -44,32 +43,6 @@ type getterStruct struct {
source string
}

// GetTerraformerSources uses Terraformer to download runtime resources from AWS provider
// to terraform.
// After Downloaded files kics scan the files as normal local files
func GetTerraformerSources(source []string, destinationPath string) (ExtractedPath, error) {
extrStruct := ExtractedPath{
Path: []string{},
ExtractionMap: make(map[string]model.ExtractedPathObject),
}

for _, path := range source {
exportedPath, err := terraformer.Import(path, destinationPath)
if err != nil {
log.Error().Msgf("failed to import %s: %s", path, err)
}

extrStruct.ExtractionMap[exportedPath] = model.ExtractedPathObject{
Path: exportedPath,
LocalPath: true,
}

extrStruct.Path = append(extrStruct.Path, exportedPath)
}

return extrStruct, nil
}

// GetKuberneterSources uses Kubernetes API to download runtime resources
// After Downloaded files kics scan the files as normal local files
func GetKuberneterSources(ctx context.Context, source []string, destinationPath string) (ExtractedPath, error) {
Expand Down
27 changes: 7 additions & 20 deletions pkg/scan/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@ import (
)

var (
terraformerRegex = regexp.MustCompile(`^terraformer::`)
kuberneterRegex = regexp.MustCompile(`^kuberneter::`)
kuberneterRegex = regexp.MustCompile(`^kuberneter::`)
)

func (c *Client) prepareAndAnalyzePaths(ctx context.Context) (provider.ExtractedPath, error) {
Expand All @@ -29,12 +28,7 @@ func (c *Client) prepareAndAnalyzePaths(ctx context.Context) (provider.Extracted
return provider.ExtractedPath{}, err
}

regularPaths, terraformerPaths, kuberneterPaths := extractPathType(c.ScanParams.Path)

terraformerExPaths, err := provider.GetTerraformerSources(terraformerPaths, c.ScanParams.OutputPath)
if err != nil {
return provider.ExtractedPath{}, err
}
regularPaths, kuberneterPaths := extractPathType(c.ScanParams.Path)

kuberneterExPaths, err := provider.GetKuberneterSources(ctx, kuberneterPaths, c.ScanParams.OutputPath)
if err != nil {
Expand All @@ -46,7 +40,7 @@ func (c *Client) prepareAndAnalyzePaths(ctx context.Context) (provider.Extracted
return provider.ExtractedPath{}, err
}

allPaths := combinePaths(terraformerExPaths, kuberneterExPaths, regularExPaths, queryExPaths, libExPaths)
allPaths := combinePaths(kuberneterExPaths, regularExPaths, queryExPaths, libExPaths)
if len(allPaths.Path) == 0 {
return provider.ExtractedPath{}, nil
}
Expand Down Expand Up @@ -78,20 +72,16 @@ func (c *Client) prepareAndAnalyzePaths(ctx context.Context) (provider.Extracted
return allPaths, nil
}

func combinePaths(terraformer, kuberneter, regular, query, library provider.ExtractedPath) provider.ExtractedPath {
func combinePaths(kuberneter, regular, query, library provider.ExtractedPath) provider.ExtractedPath {
var combinedPaths provider.ExtractedPath
paths := make([]string, 0)
combinedPathsEx := make(map[string]model.ExtractedPathObject)
paths = append(paths, terraformer.Path...)
paths = append(paths, kuberneter.Path...)
paths = append(paths, regular.Path...)
combinedPaths.Path = paths
for k, v := range regular.ExtractionMap {
combinedPathsEx[k] = v
}
for k, v := range terraformer.ExtractionMap {
combinedPathsEx[k] = v
}
for k, v := range kuberneter.ExtractionMap {
combinedPathsEx[k] = v
}
Expand Down Expand Up @@ -208,11 +198,9 @@ func logLoadingQueriesType(types []string) {
log.Info().Msgf("Loading queries of type: %s", strings.Join(types, ", "))
}

func extractPathType(paths []string) (regular, terraformer, kuberneter []string) {
func extractPathType(paths []string) (regular, kuberneter []string) {
for _, path := range paths {
if terraformerRegex.MatchString(path) {
terraformer = append(terraformer, terraformerRegex.ReplaceAllString(path, ""))
} else if kuberneterRegex.MatchString(path) {
if kuberneterRegex.MatchString(path) {
kuberneter = append(kuberneter, kuberneterRegex.ReplaceAllString(path, ""))
} else {
regular = append(regular, path)
Expand All @@ -223,8 +211,7 @@ func extractPathType(paths []string) (regular, terraformer, kuberneter []string)

func deleteExtractionFolder(extractionMap map[string]model.ExtractedPathObject) {
for extractionFile := range extractionMap {
if strings.Contains(extractionFile, "kics-extract-terraformer") ||
strings.Contains(extractionFile, "kics-extract-kuberneter") {
if strings.Contains(extractionFile, "kics-extract-kuberneter") {
continue
}
err := os.RemoveAll(extractionFile)
Expand Down
Loading
Loading